From 950ab820b1f88eccc8a3585ac84954082789edb9 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 19:48:38 -0400 Subject: [PATCH 01/72] test: make bare pytest collect and skip tests whose optional extras are absent Add pythonpath = . to pytest.ini so tests/conftest.py can import the tests package under a bare pytest run, and drop the duplicate [tool.pytest.ini_options] block from pyproject.toml. Guard modules that import numpy or rank_bm25 at import time with module-level pytest.importorskip, and mark only the individual tests that need tiktoken, numpy, huggingface_hub, the models-local profile or the E2B SDK, so the rest of those modules keeps running under CI's [dev,apps,graph] install. Also apply ruff format to tests/test_retrieval_runtime.py. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 6 ------ pytest.ini | 1 + tests/api_v7/test_external_operations.py | 5 +++++ tests/test_auto_zoom_integration.py | 2 ++ tests/test_bm25.py | 4 ++++ tests/test_compression.py | 26 ++++++++++++++++++++++++ tests/test_compression_integration.py | 15 ++++++++++++++ tests/test_contradiction.py | 12 +++++++++++ tests/test_fusion.py | 2 ++ tests/test_hybrid_integration.py | 2 ++ tests/test_query_classifier.py | 5 ++++- tests/test_reflexion_integration.py | 10 +++++++++ tests/test_retrieval_router.py | 2 ++ tests/test_retrieval_router_v7.py | 4 ++++ tests/test_retrieval_runtime.py | 8 +++++++- tests/test_surprise_calc.py | 4 ++++ tests/test_vectors.py | 9 ++++++++ tests/test_verification.py | 11 ++++++++++ 18 files changed, 120 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8644a02..e60eed9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,9 +122,3 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false -[tool.pytest.ini_options] -asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" -testpaths = ["tests"] -python_files = ["test_*.py"] -python_functions = ["test_*"] diff --git a/pytest.ini b/pytest.ini index 3e5daee..966704f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ asyncio_mode = auto asyncio_default_fixture_loop_scope = function testpaths = tests +pythonpath = . python_files = test_*.py python_functions = test_* cache_dir = .test_tmp/pytest_cache diff --git a/tests/api_v7/test_external_operations.py b/tests/api_v7/test_external_operations.py index e9ed242..1a1126d 100644 --- a/tests/api_v7/test_external_operations.py +++ b/tests/api_v7/test_external_operations.py @@ -7,6 +7,7 @@ import unittest from contextlib import closing from datetime import datetime, timezone +from importlib.util import find_spec from pathlib import Path from types import MappingProxyType from unittest.mock import ANY, AsyncMock, patch @@ -293,6 +294,10 @@ async def test_ingest_key_rebinding_is_atomic_conflict(self) -> None: self.assertEqual(event_ids, original.event_ids) +@unittest.skipIf( + find_spec("e2b_code_interpreter") is None, + "e2b_code_interpreter ([agency-e2b] extra) is not installed", +) class ModernE2BProviderTests(unittest.IsolatedAsyncioTestCase): async def test_real_sdk_result_shape_is_bounded_and_sandbox_is_isolated( self, diff --git a/tests/test_auto_zoom_integration.py b/tests/test_auto_zoom_integration.py index 4201391..3b9ad18 100644 --- a/tests/test_auto_zoom_integration.py +++ b/tests/test_auto_zoom_integration.py @@ -25,6 +25,8 @@ import pytest +pytest.importorskip("numpy") + from daem0nmcp.compression.jit import JITCompressor from daem0nmcp.recall_planner import QueryComplexity from daem0nmcp.retrieval_router import RetrievalRouter diff --git a/tests/test_bm25.py b/tests/test_bm25.py index 88eb7a2..e2012cc 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -1,6 +1,10 @@ # tests/test_bm25.py """Tests for BM25 index.""" +import pytest + +pytest.importorskip("rank_bm25") + from daem0nmcp.bm25_index import BM25Index diff --git a/tests/test_compression.py b/tests/test_compression.py index 3b1597e..3ca47db 100644 --- a/tests/test_compression.py +++ b/tests/test_compression.py @@ -1,9 +1,16 @@ """Tests for compression module.""" +from importlib.util import find_spec + import pytest from daem0nmcp.compression import CompressionConfig, ContextCompressor +requires_tiktoken = pytest.mark.skipif( + find_spec("tiktoken") is None, + reason="tiktoken ([models-hosted] extra) is not installed", +) + class TestCompressionConfig: """Tests for CompressionConfig.""" @@ -55,6 +62,7 @@ def test_lazy_initialization(self): compressor = ContextCompressor() assert compressor._compressor is None + @requires_tiktoken def test_count_tokens_short_text(self): """Token counting works for short text.""" compressor = ContextCompressor() @@ -62,24 +70,28 @@ def test_count_tokens_short_text(self): assert count > 0 assert count < 10 # Should be 2-3 tokens + @requires_tiktoken def test_count_tokens_empty_string(self): """Token counting works for empty string.""" compressor = ContextCompressor() count = compressor.count_tokens("") assert count == 0 + @requires_tiktoken def test_count_tokens_unicode(self): """Token counting handles unicode text.""" compressor = ContextCompressor() count = compressor.count_tokens("Hello") assert count > 0 + @requires_tiktoken def test_should_compress_under_threshold(self): """should_compress returns False for short text.""" compressor = ContextCompressor() short_text = "This is a short text." assert not compressor.should_compress(short_text) + @requires_tiktoken def test_should_compress_over_threshold(self): """should_compress returns True for long text.""" config = CompressionConfig( @@ -89,6 +101,7 @@ def test_should_compress_over_threshold(self): long_text = "This is a longer text that should exceed the threshold. " * 10 assert compressor.should_compress(long_text) + @requires_tiktoken def test_compress_skips_under_threshold(self): """compress() returns unchanged text when under threshold.""" compressor = ContextCompressor() @@ -99,6 +112,7 @@ def test_compress_skips_under_threshold(self): assert result["skipped"] is True assert result["ratio"] == 1.0 + @requires_tiktoken def test_compress_returns_all_fields(self): """compress() returns all expected fields.""" compressor = ContextCompressor() @@ -111,6 +125,7 @@ def test_compress_returns_all_fields(self): assert "ratio" in result assert "skipped" in result + @requires_tiktoken def test_compress_simple_returns_string(self): """compress_simple returns just the text.""" compressor = ContextCompressor() @@ -126,6 +141,7 @@ def test_config_passed_to_compressor(self): compressor = ContextCompressor(config) assert compressor.config.compression_threshold == 100 + @requires_tiktoken def test_tokenizer_lazy_loaded_on_count(self): """Tokenizer is loaded lazily on first count_tokens call.""" compressor = ContextCompressor() @@ -204,6 +220,7 @@ def test_get_rate_for_narrative(self): rate = adaptive.get_rate_for_content(ContentType.NARRATIVE) assert rate == 0.2 # 5x compression + @requires_tiktoken def test_compress_includes_content_type(self): """Compression result includes detected content type.""" from daem0nmcp.compression import AdaptiveCompressor @@ -214,6 +231,7 @@ def test_compress_includes_content_type(self): assert "content_type" in result assert result["content_type"] == "narrative" + @requires_tiktoken def test_compress_simple_returns_string(self): """compress_simple returns just the text.""" from daem0nmcp.compression import AdaptiveCompressor @@ -227,6 +245,7 @@ def test_compress_simple_returns_string(self): class TestContextCompressorIntegration: """Integration tests that require model loading.""" + @requires_tiktoken @pytest.mark.slow def test_compress_actually_compresses(self): """Compression achieves target ratio (requires model).""" @@ -242,6 +261,7 @@ def test_compress_actually_compresses(self): assert result["compressed_tokens"] < result["original_tokens"] assert result["ratio"] > 1.0 + @requires_tiktoken @pytest.mark.slow def test_force_tokens_preserved(self): """Force tokens are preserved in compressed output.""" @@ -260,6 +280,7 @@ def test_force_tokens_preserved(self): class TestHierarchicalContextManager: """Tests for HierarchicalContextManager.""" + @requires_tiktoken def test_simple_query_uses_summaries(self): """Simple queries prefer community summaries.""" from daem0nmcp.compression import HierarchicalContextManager @@ -278,6 +299,7 @@ def test_simple_query_uses_summaries(self): assert result["compression_applied"] is False assert "Auth system" in result["context"] + @requires_tiktoken def test_simple_query_fallback_to_raw(self): """Simple queries fall back to raw if no summaries.""" from daem0nmcp.compression import HierarchicalContextManager @@ -294,6 +316,7 @@ def test_simple_query_fallback_to_raw(self): assert result["strategy"] == "raw_fallback" assert "Raw memory content" in result["context"] + @requires_tiktoken def test_complex_query_uses_compression(self): """Complex queries use compression strategy.""" from daem0nmcp.compression import HierarchicalContextManager @@ -337,6 +360,7 @@ def test_format_summaries(self): assert "Summary A" in formatted assert "Summary B" in formatted + @requires_tiktoken def test_result_includes_token_count(self): """All results include token count.""" from daem0nmcp.compression import HierarchicalContextManager @@ -353,6 +377,7 @@ def test_result_includes_token_count(self): assert "token_count" in result assert isinstance(result["token_count"], int) + @requires_tiktoken def test_medium_query_hybrid_strategy(self): """Medium complexity queries use hybrid strategy.""" from daem0nmcp.compression import HierarchicalContextManager @@ -370,6 +395,7 @@ def test_medium_query_hybrid_strategy(self): # Should be hybrid or hybrid_compressed assert result["strategy"] in ["hybrid", "hybrid_compressed"] + @requires_tiktoken def test_empty_memories_handled(self): """Empty memory list is handled gracefully.""" from daem0nmcp.compression import HierarchicalContextManager diff --git a/tests/test_compression_integration.py b/tests/test_compression_integration.py index e446178..3334c0e 100644 --- a/tests/test_compression_integration.py +++ b/tests/test_compression_integration.py @@ -9,6 +9,15 @@ - CONTEXT-05: Adaptive compression ratios based on query complexity """ +from importlib.util import find_spec + +import pytest + +requires_tiktoken = pytest.mark.skipif( + find_spec("tiktoken") is None, + reason="tiktoken ([models-hosted] extra) is not installed", +) + class TestContextEngineeringRequirements: """Integration tests verifying Phase 4 requirements.""" @@ -92,6 +101,7 @@ def apply(self, price): assert "calculate_discount" in names assert "DiscountCalculator" in names + @requires_tiktoken def test_context_04_hierarchical_summaries(self): """CONTEXT-04: Hierarchical compression leverages community structure.""" from daem0nmcp.compression import HierarchicalContextManager @@ -179,6 +189,7 @@ def test_context_05_recall_plan_compression(self): class TestCompressionThreshold: """Tests for 4K token compression threshold.""" + @requires_tiktoken def test_under_threshold_not_compressed(self): """Context under 4K tokens is not compressed.""" from daem0nmcp.compression import ContextCompressor @@ -192,6 +203,7 @@ def test_under_threshold_not_compressed(self): assert result["skipped"] is True assert result["compressed_prompt"] == short_text + @requires_tiktoken def test_threshold_configurable(self): """Compression threshold is configurable.""" from daem0nmcp.compression import CompressionConfig, ContextCompressor @@ -207,6 +219,7 @@ def test_threshold_configurable(self): class TestHierarchicalStrategies: """Tests for hierarchical context strategies.""" + @requires_tiktoken def test_simple_strategy_prefers_summaries(self): """Simple queries prefer community summaries.""" from daem0nmcp.compression import HierarchicalContextManager @@ -221,6 +234,7 @@ def test_simple_strategy_prefers_summaries(self): assert result["strategy"] == "summaries" + @requires_tiktoken def test_complex_strategy_compresses_raw(self): """Complex queries compress raw memories.""" from daem0nmcp.compression import HierarchicalContextManager @@ -236,6 +250,7 @@ def test_complex_strategy_compresses_raw(self): # Should attempt compression strategy assert result["strategy"] in ["compressed", "raw"] + @requires_tiktoken def test_medium_strategy_hybrid(self): """Medium queries use hybrid approach.""" from daem0nmcp.compression import HierarchicalContextManager diff --git a/tests/test_contradiction.py b/tests/test_contradiction.py index d827d53..4f6f43c 100644 --- a/tests/test_contradiction.py +++ b/tests/test_contradiction.py @@ -3,6 +3,7 @@ import shutil import tempfile from datetime import datetime, timezone +from importlib.util import find_spec import pytest @@ -15,6 +16,11 @@ ) from daem0nmcp.models import Memory, MemoryVersion +requires_models_local = pytest.mark.skipif( + find_spec("sentence_transformers") is None, + reason="models-local profile is not installed", +) + @pytest.fixture(autouse=True) def models_local_enabled(monkeypatch): @@ -148,6 +154,7 @@ def test_contradiction_default_reason(self): assert c.negation_pattern is None +@requires_models_local @pytest.mark.asyncio async def test_detect_contradictions_high_similarity_with_negation(db_manager): """High similarity + negation pattern = contradiction detected.""" @@ -184,6 +191,7 @@ async def test_detect_contradictions_high_similarity_with_negation(db_manager): assert contradictions[0].negation_pattern is not None +@requires_models_local @pytest.mark.asyncio async def test_detect_contradictions_no_contradiction_without_negation(db_manager): """High similarity without negation = no contradiction.""" @@ -217,6 +225,7 @@ async def test_detect_contradictions_no_contradiction_without_negation(db_manage assert len(contradictions) == 0 +@requires_models_local @pytest.mark.asyncio async def test_detect_contradictions_skips_invalidated_versions(db_manager): """Already invalidated versions should not be checked for contradictions.""" @@ -250,6 +259,7 @@ async def test_detect_contradictions_skips_invalidated_versions(db_manager): assert len(contradictions) == 0 +@requires_models_local @pytest.mark.asyncio async def test_detect_contradictions_excludes_same_memory(db_manager): """Can exclude versions from the same memory (avoid self-contradiction).""" @@ -284,6 +294,7 @@ async def test_detect_contradictions_excludes_same_memory(db_manager): assert len(contradictions) == 0 +@requires_models_local @pytest.mark.asyncio async def test_detect_contradictions_low_similarity_no_contradiction(db_manager): """Low similarity = no contradiction even with negation patterns.""" @@ -437,6 +448,7 @@ async def test_invalidate_skips_already_invalidated(db_manager): assert version.valid_to.day == 1 +@requires_models_local @pytest.mark.asyncio async def test_check_and_invalidate_combined(db_manager): """check_and_invalidate_contradictions combines detection and invalidation.""" diff --git a/tests/test_fusion.py b/tests/test_fusion.py index 356c574..3fb90d0 100644 --- a/tests/test_fusion.py +++ b/tests/test_fusion.py @@ -3,6 +3,8 @@ import pytest +pytest.importorskip("rank_bm25") + from daem0nmcp.fusion import RRFHybridSearch, reciprocal_rank_fusion diff --git a/tests/test_hybrid_integration.py b/tests/test_hybrid_integration.py index a54904f..cae0925 100644 --- a/tests/test_hybrid_integration.py +++ b/tests/test_hybrid_integration.py @@ -3,6 +3,8 @@ import pytest +pytest.importorskip("rank_bm25") + from daem0nmcp.bm25_index import BM25Index from daem0nmcp.fusion import RRFHybridSearch from daem0nmcp.vectors import VectorIndex diff --git a/tests/test_query_classifier.py b/tests/test_query_classifier.py index 3bd528f..d892367 100644 --- a/tests/test_query_classifier.py +++ b/tests/test_query_classifier.py @@ -8,9 +8,12 @@ from unittest.mock import MagicMock, patch -import numpy as np import pytest +pytest.importorskip("numpy") + +import numpy as np + from daem0nmcp.query_classifier import ExemplarQueryClassifier, QueryComplexity # --------------------------------------------------------------------------- diff --git a/tests/test_reflexion_integration.py b/tests/test_reflexion_integration.py index 1d1a1d3..b688017 100644 --- a/tests/test_reflexion_integration.py +++ b/tests/test_reflexion_integration.py @@ -4,6 +4,7 @@ work together: claim extraction -> verification -> persistence -> consolidation. """ +from importlib.util import find_spec from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -28,6 +29,11 @@ verify_claims, ) +requires_models_local = pytest.mark.skipif( + find_spec("sentence_transformers") is None, + reason="models-local profile is not installed", +) + @pytest.fixture(autouse=True) def models_local_enabled(monkeypatch): @@ -122,6 +128,7 @@ async def test_memory_claim_verification_pipeline(self, mock_memory_manager): assert pg_result is not None assert pg_result.status == "verified" + @requires_models_local @pytest.mark.asyncio async def test_unverified_claim_no_evidence(self, mock_memory_manager): """Claims with no supporting memories should be unverified.""" @@ -514,6 +521,7 @@ async def mock_recall_reflections(**kwargs): class TestVerifyFactsToolIntegration: """Tests for verify_facts MCP tool (via direct import).""" + @requires_models_local @pytest.mark.asyncio async def test_verify_facts_returns_structure(self, mock_memory_manager): """Test verify_facts returns expected structure.""" @@ -592,6 +600,7 @@ async def test_no_claims_returns_empty_structure(self, mock_memory_manager): class TestGraphRAGIntegration: """Tests for GraphRAG integration in verification.""" + @requires_models_local @pytest.mark.asyncio async def test_verification_with_knowledge_graph( self, mock_memory_manager, mock_knowledge_graph @@ -616,6 +625,7 @@ async def test_verification_with_knowledge_graph( # Should have called ensure_loaded on knowledge graph mock_knowledge_graph.ensure_loaded.assert_called() + @requires_models_local @pytest.mark.asyncio async def test_verification_without_knowledge_graph(self, mock_memory_manager): """Test verification works without knowledge graph.""" diff --git a/tests/test_retrieval_router.py b/tests/test_retrieval_router.py index 53260b1..3a854a4 100644 --- a/tests/test_retrieval_router.py +++ b/tests/test_retrieval_router.py @@ -11,6 +11,8 @@ import pytest +pytest.importorskip("numpy") + from daem0nmcp.recall_planner import QueryComplexity from daem0nmcp.retrieval_router import RetrievalRouter diff --git a/tests/test_retrieval_router_v7.py b/tests/test_retrieval_router_v7.py index d8e184e..bc19497 100644 --- a/tests/test_retrieval_router_v7.py +++ b/tests/test_retrieval_router_v7.py @@ -6,6 +6,10 @@ import types import unittest +import pytest + +pytest.importorskip("numpy") + _previous_package = sys.modules.get("sentence_transformers") _previous_util = sys.modules.get("sentence_transformers.util") _util = types.ModuleType("sentence_transformers.util") diff --git a/tests/test_retrieval_runtime.py b/tests/test_retrieval_runtime.py index 3e75808..3a537ba 100644 --- a/tests/test_retrieval_runtime.py +++ b/tests/test_retrieval_runtime.py @@ -12,6 +12,7 @@ import types import unittest from datetime import datetime, timezone +from importlib.util import find_spec from pathlib import Path from types import SimpleNamespace @@ -435,6 +436,9 @@ def test_include_warnings_does_not_narrow_an_unfiltered_legacy_recall(self): with self.assertRaises(ValueError): normalize_legacy_category_filter(1, True) + @unittest.skipIf( + find_spec("huggingface_hub") is None, "huggingface_hub is not installed" + ) def test_configured_embedding_backend_never_silently_changes_contract(self): from unittest.mock import patch @@ -707,7 +711,9 @@ def blocking_deadline(_path): self.assertTrue(finished.is_set()) - async def test_database_close_defers_repeated_cancellation_until_drain_cleanup(self): + async def test_database_close_defers_repeated_cancellation_until_drain_cleanup( + self, + ): from unittest.mock import patch from daem0nmcp.database import DatabaseManager diff --git a/tests/test_surprise_calc.py b/tests/test_surprise_calc.py index 6a2156b..eaa01b5 100644 --- a/tests/test_surprise_calc.py +++ b/tests/test_surprise_calc.py @@ -1,6 +1,10 @@ # tests/test_surprise_calc.py """Tests for surprise score calculation.""" +import pytest + +pytest.importorskip("numpy") + from daem0nmcp.surprise import SurpriseCalculator, calculate_surprise diff --git a/tests/test_vectors.py b/tests/test_vectors.py index f027236..ac78d0e 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -1,5 +1,6 @@ """Tests for the vector embeddings module.""" +from importlib.util import find_spec from unittest.mock import MagicMock, patch import pytest @@ -7,6 +8,11 @@ from daem0nmcp import vectors from daem0nmcp.config import settings +requires_numpy = pytest.mark.skipif( + find_spec("numpy") is None, + reason="numpy ([models-local] extra) is not installed", +) + class TestVectorAvailability: """Test vector availability detection.""" @@ -116,6 +122,7 @@ def test_reset_clears_index(self): class TestEncodeQueryDocument: """Test encode_query and encode_document use correct prefixes.""" + @requires_numpy @patch("daem0nmcp.vectors._get_model") def test_encode_document_prepends_document_prefix(self, mock_get_model): import numpy as np @@ -134,6 +141,7 @@ def test_encode_document_prepends_document_prefix(self, mock_get_model): call_text = fake_model.encode.call_args[0][0] assert call_text == f"{settings.embedding_document_prefix}hello world" + @requires_numpy @patch("daem0nmcp.vectors._get_model") def test_encode_query_prepends_query_prefix(self, mock_get_model): import numpy as np @@ -150,6 +158,7 @@ def test_encode_query_prepends_query_prefix(self, mock_get_model): call_text = fake_model.encode.call_args[0][0] assert call_text == f"{settings.embedding_query_prefix}hello world" + @requires_numpy @patch("daem0nmcp.vectors._get_model") def test_encode_returns_correct_byte_length(self, mock_get_model): import numpy as np diff --git a/tests/test_verification.py b/tests/test_verification.py index bf43483..b6ea8cf 100644 --- a/tests/test_verification.py +++ b/tests/test_verification.py @@ -7,6 +7,7 @@ - Summary aggregation """ +from importlib.util import find_spec from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -20,6 +21,11 @@ verify_claims, ) +requires_models_local = pytest.mark.skipif( + find_spec("sentence_transformers") is None, + reason="models-local profile is not installed", +) + @pytest.fixture(autouse=True) def models_local_enabled(monkeypatch): @@ -121,6 +127,7 @@ async def test_verify_against_memory_found( assert len(result.evidence) >= 1 assert result.evidence[0].source == "memory" + @requires_models_local @pytest.mark.asyncio async def test_verify_no_evidence_unverified( self, mock_memory_manager, memory_reference_claim @@ -178,6 +185,7 @@ async def test_verify_conflict_detected(self, mock_memory_manager): assert result.conflict_reason is not None assert "negation" in result.conflict_reason.lower() + @requires_models_local @pytest.mark.asyncio async def test_verify_with_knowledge_graph( self, mock_memory_manager, mock_knowledge_graph, memory_reference_claim @@ -216,6 +224,7 @@ async def test_verify_skip_level_auto_verified( # Memory manager should not have been called for skip-level mock_memory_manager.recall.assert_not_called() + @requires_models_local @pytest.mark.asyncio async def test_verify_with_as_of_time( self, mock_memory_manager, memory_reference_claim @@ -277,6 +286,7 @@ async def test_verify_low_similarity_not_evidence( class TestVerifyClaims: """Tests for verify_claims function.""" + @requires_models_local @pytest.mark.asyncio async def test_verify_multiple_claims(self, mock_memory_manager): """Multiple claims should all be verified.""" @@ -317,6 +327,7 @@ async def test_verify_empty_claims_list(self, mock_memory_manager): assert len(results) == 0 + @requires_models_local @pytest.mark.asyncio async def test_verify_claims_preserves_order(self, mock_memory_manager): """Results should be in same order as input claims.""" From 946c55a371e122f2c7ad20538578f89e63d14a51 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 19:48:38 -0400 Subject: [PATCH 02/72] fix: use sys.platform guards so mypy type-checks Windows-only code on Linux mypy narrows sys.platform but not os.name, so the ctypes.WinDLL and msvcrt.locking calls failed the Linux type-check job. Declare the job handle closer's type instead of ignoring the attribute error. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/resource_repository.py | 10 ++++++---- daem0nmcp/api/v7/workspace_bootstrap.py | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index 03c1f31..c1c7a07 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -17,6 +17,7 @@ import re import sqlite3 import subprocess +import sys import tempfile from collections.abc import Awaitable, Callable, Mapping from contextlib import suppress @@ -46,7 +47,8 @@ class _WindowsKillOnCloseJob: def __init__(self, process: subprocess.Popen[bytes]) -> None: self._handle: object | None = None - if os.name != "nt": + self._close: Callable[[object], object] | None = None + if sys.platform != "win32": return import ctypes from ctypes import wintypes @@ -110,7 +112,7 @@ class _ExtendedLimit(ctypes.Structure): def resume(self, process_id: int) -> None: """Resume the primary thread only after the suspended process is owned.""" - if os.name != "nt": + if sys.platform != "win32": return import ctypes from ctypes import wintypes @@ -167,8 +169,8 @@ class ThreadEntry(ctypes.Structure): kernel32.CloseHandle(snapshot) def close(self) -> None: - if self._handle is not None: - self._close(self._handle) # type: ignore[attr-defined] + if self._handle is not None and self._close is not None: + self._close(self._handle) self._handle = None diff --git a/daem0nmcp/api/v7/workspace_bootstrap.py b/daem0nmcp/api/v7/workspace_bootstrap.py index d0eacce..c4c93c7 100644 --- a/daem0nmcp/api/v7/workspace_bootstrap.py +++ b/daem0nmcp/api/v7/workspace_bootstrap.py @@ -8,6 +8,7 @@ import secrets import sqlite3 import stat +import sys import tempfile from contextlib import suppress from pathlib import Path @@ -95,7 +96,7 @@ def _open_bootstrap_marker(storage: Path) -> int: raise ProtectedPathError("bootstrap marker is not a regular file") if not os.path.samestat(opened, named): raise ProtectedPathError("bootstrap marker identity changed") - if os.name == "nt": + if sys.platform == "win32": import msvcrt try: From c483bc1ab2311f1f4d7c2df290d7633504e623a1 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 19:48:38 -0400 Subject: [PATCH 03/72] ci: pin ruff to the locked version and document the CI gates Pin ruff 0.16.8 (uv.lock) in the lint job and the pre-commit hook, and drop the continue-on-error legacy mypy step. Rewrite the AGENTS.md contributor commands to the gates ci.yml actually runs. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 7 +------ .pre-commit-config.yaml | 2 +- AGENTS.md | 13 ++++++++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c06db9..4a5ad75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ruff + pip install ruff==0.16.8 # keep in sync with uv.lock and .pre-commit-config.yaml - name: Lint with ruff run: | @@ -101,8 +101,3 @@ jobs: - name: Type check v7 production surface run: | mypy daem0nmcp/api/v7 --ignore-missing-imports --follow-imports=silent - - - name: Report legacy repository type errors - continue-on-error: true - run: | - mypy daem0nmcp/ --ignore-missing-imports diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c14a1d3..773d3d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: # Ruff - fast Python linter and formatter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.0 + rev: v0.16.8 hooks: - id: ruff args: [--fix] diff --git a/AGENTS.md b/AGENTS.md index c242d0f..4c90e08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,19 +9,22 @@ - Runtime data lives under `.daem0nmcp/` (e.g., `.daem0nmcp/storage/daem0nmcp.db`); do not commit it. ## Build, Test, and Development Commands -- `pip install -e ".[dev]"` installs the package in editable mode with test deps. +- `pip install -e ".[dev,apps,graph]"` installs the package in editable mode with the extras CI tests against. - `python -m daem0nmcp.server` runs the MCP server directly. - `python start_server.py --port 9876` starts the Windows HTTP launcher. - `python -m daem0nmcp.cli ` runs CLI tasks (example: `python -m daem0nmcp.cli index`). -- `pytest tests/ -v --asyncio-mode=auto` runs the test suite. -- `ruff check daem0nmcp/ tests/` runs the CI lint step. -- `mypy daem0nmcp/ --ignore-missing-imports` runs the optional type check (CI does not fail on errors). + +CI (`.github/workflows/ci.yml`) blocks merges on these gates; run them before pushing: +- Tests: `pytest tests/ -v --asyncio-mode=auto` (Ubuntu, Windows and macOS on Python 3.10-3.12). +- Lint and format: `ruff check daem0nmcp/ tests/` and `ruff format --check daem0nmcp/ tests/`, with ruff pinned to the `uv.lock` version (0.16.8). +- Type check: `mypy daem0nmcp/api/v7 --ignore-missing-imports --follow-imports=silent` (CI runs it on Linux; add `--platform linux` locally on Windows or macOS). +- Release inventory: `python scripts/v7_release_inventory.py --check`. ## Coding Style & Naming Conventions - Use 4-space indentation and follow PEP 8 layout. - `snake_case` for functions/variables, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants. - Keep modules focused and add new features under `daem0nmcp/` with corresponding tests. -- Aim for lint-clean code under `ruff check`; no formatter is enforced, so match nearby style. +- Code must pass `ruff check` and `ruff format --check`. ## Testing Guidelines - Tests use `pytest` with `pytest-asyncio`; stick to `test_*.py` and `test_*` names. From ce495802f29281054f9a7de7635129639073d3a0 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 19:48:38 -0400 Subject: [PATCH 04/72] fix(deps): refresh locked mcp, cryptography, urllib3 and idna mcp 1.26.0 -> 1.30.0, cryptography 46.0.3 -> 50.0.1, urllib3 2.6.1 -> 2.8.0 and idna 3.11 -> 3.20 clear the published advisories against the locked versions. httpx is unchanged. Co-Authored-By: Claude Opus 5 --- uv.lock | 122 ++++++++++++++++++++++++++------------------------------ 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/uv.lock b/uv.lock index 92d1705..1236282 100644 --- a/uv.lock +++ b/uv.lock @@ -529,67 +529,59 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.3" +version = "50.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/42/9c391dd801d6cf0d561b5890549d4b27bafcc53b39c31a817e69d87c625b/cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a", size = 7225004, upload-time = "2025-10-15T23:16:52.239Z" }, - { url = "https://files.pythonhosted.org/packages/1c/67/38769ca6b65f07461eb200e85fc1639b438bdc667be02cf7f2cd6a64601c/cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc", size = 4296667, upload-time = "2025-10-15T23:16:54.369Z" }, - { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" }, - { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" }, - { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" }, - { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" }, - { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" }, - { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" }, - { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" }, - { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" }, - { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" }, - { url = "https://files.pythonhosted.org/packages/96/92/8a6a9525893325fc057a01f654d7efc2c64b9de90413adcf605a85744ff4/cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018", size = 3055988, upload-time = "2025-10-15T23:17:14.65Z" }, - { url = "https://files.pythonhosted.org/packages/7e/bf/80fbf45253ea585a1e492a6a17efcb93467701fa79e71550a430c5e60df0/cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb", size = 3514451, upload-time = "2025-10-15T23:17:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/2e/af/9b302da4c87b0beb9db4e756386a7c6c5b8003cd0e742277888d352ae91d/cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c", size = 2928007, upload-time = "2025-10-15T23:17:18.04Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e2/a510aa736755bffa9d2f75029c229111a1d02f8ecd5de03078f4c18d91a3/cryptography-46.0.3-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217", size = 7158012, upload-time = "2025-10-15T23:17:19.982Z" }, - { url = "https://files.pythonhosted.org/packages/73/dc/9aa866fbdbb95b02e7f9d086f1fccfeebf8953509b87e3f28fff927ff8a0/cryptography-46.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5", size = 4288728, upload-time = "2025-10-15T23:17:21.527Z" }, - { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" }, - { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" }, - { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" }, - { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" }, - { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" }, - { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" }, - { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" }, - { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" }, - { url = "https://files.pythonhosted.org/packages/fd/30/27654c1dbaf7e4a3531fa1fc77986d04aefa4d6d78259a62c9dc13d7ad36/cryptography-46.0.3-cp314-cp314t-win32.whl", hash = "sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914", size = 3022339, upload-time = "2025-10-15T23:17:40.888Z" }, - { url = "https://files.pythonhosted.org/packages/f6/30/640f34ccd4d2a1bc88367b54b926b781b5a018d65f404d409aba76a84b1c/cryptography-46.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db", size = 3494315, upload-time = "2025-10-15T23:17:42.769Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8b/88cc7e3bd0a8e7b861f26981f7b820e1f46aa9d26cc482d0feba0ecb4919/cryptography-46.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21", size = 2919331, upload-time = "2025-10-15T23:17:44.468Z" }, - { url = "https://files.pythonhosted.org/packages/fd/23/45fe7f376a7df8daf6da3556603b36f53475a99ce4faacb6ba2cf3d82021/cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936", size = 7218248, upload-time = "2025-10-15T23:17:46.294Z" }, - { url = "https://files.pythonhosted.org/packages/27/32/b68d27471372737054cbd34c84981f9edbc24fe67ca225d389799614e27f/cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683", size = 4294089, upload-time = "2025-10-15T23:17:48.269Z" }, - { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" }, - { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" }, - { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" }, - { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" }, - { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" }, - { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" }, - { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" }, - { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" }, - { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" }, - { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" }, - { url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" }, - { url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" }, - { url = "https://files.pythonhosted.org/packages/99/55/181022996c4063fc0e7666a47049a1ca705abb9c8a13830f074edb347495/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717", size = 4242957, upload-time = "2025-10-15T23:18:22.18Z" }, - { url = "https://files.pythonhosted.org/packages/ba/af/72cd6ef29f9c5f731251acadaeb821559fe25f10852f44a63374c9ca08c1/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9", size = 4409447, upload-time = "2025-10-15T23:18:24.209Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c3/e90f4a4feae6410f914f8ebac129b9ae7a8c92eb60a638012dde42030a9d/cryptography-46.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c", size = 3438528, upload-time = "2025-10-15T23:18:26.227Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/19/797e2aaac9df6a66f1550f49979dc1b1e39ecd2077501c30efa81e8d5d67/cryptography-50.0.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986", size = 4010153, upload-time = "2026-08-25T19:44:03.155Z" }, + { url = "https://files.pythonhosted.org/packages/90/34/9ce9a62ed9dc82ca9fd6a34445b6904af56e5f38b3eae2ed32e49c36053d/cryptography-50.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f", size = 4723133, upload-time = "2026-08-25T19:44:05.461Z" }, + { url = "https://files.pythonhosted.org/packages/57/26/e6d4fc8512a51a5f9ee7bfdbfb853bce1197087df40c9ad993ad370b846f/cryptography-50.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef", size = 4712478, upload-time = "2026-08-25T19:44:07.375Z" }, + { url = "https://files.pythonhosted.org/packages/e6/de/d3cdc2815697aae84126cbd6a030ca7b6b452e28a88b501b836bd3aa7a86/cryptography-50.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8", size = 4730726, upload-time = "2026-08-25T19:44:09.294Z" }, + { url = "https://files.pythonhosted.org/packages/55/32/38c0d344b98c06d34b5df8946565a9c0d6dbf32c8e0730a7f05f0a3c6cab/cryptography-50.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45", size = 5353524, upload-time = "2026-08-25T19:44:11.96Z" }, + { url = "https://files.pythonhosted.org/packages/e1/1b/82f0f0d8858d4432be1af790477edf62aef90324041aa07c57e57bef1af7/cryptography-50.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad", size = 4746720, upload-time = "2026-08-25T19:44:14.051Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/042ca458b8c64348c768284b5d23e69b92ed53d057ab779fee628564676d/cryptography-50.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49", size = 4361866, upload-time = "2026-08-25T19:44:16.167Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/e96c1ef71edef71057c7e3c3d982ce8fda554e0c52d0cc19c18845cde3eb/cryptography-50.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f", size = 4730028, upload-time = "2026-08-25T19:44:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/e3/38/45abd72ef63f2e7d0754a6cacf97bd8b69512ace7f6130d24c39ece65da2/cryptography-50.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527", size = 5308405, upload-time = "2026-08-25T19:44:20.197Z" }, + { url = "https://files.pythonhosted.org/packages/85/66/6ccca4722987ddedaa7fc9c3f4708af7431f5535666c174350830888c6b7/cryptography-50.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a", size = 4746230, upload-time = "2026-08-25T19:44:22.376Z" }, + { url = "https://files.pythonhosted.org/packages/13/0e/b1f92e013228111413f2e6743948b80bc24dfd3c1b87ba98ceea16f5df89/cryptography-50.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959", size = 4862596, upload-time = "2026-08-25T19:44:24.472Z" }, + { url = "https://files.pythonhosted.org/packages/7e/22/c3654cccc856e9d682817b04ac3ee79731cb09ca6f95996a95c904de2883/cryptography-50.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b", size = 5014082, upload-time = "2026-08-25T19:44:26.709Z" }, + { url = "https://files.pythonhosted.org/packages/42/8b/cb12b1b60c91b074ca6bf0fdd59aa8f10d8bc5f73af8faece86ef0421b37/cryptography-50.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648", size = 3842826, upload-time = "2026-08-25T19:44:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f0/424cb557d99aa86ac55da5e2add02e2882e44047b6264f93ade1b975a993/cryptography-50.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f", size = 3973525, upload-time = "2026-08-25T19:44:30.7Z" }, + { url = "https://files.pythonhosted.org/packages/4d/72/3a2711d967977ab5fc80b782837c7e8d1ac7445e764c20c381a265c57ef3/cryptography-50.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a", size = 4708817, upload-time = "2026-08-25T19:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f2/bb1f56e10815b789df0b409a69fa4992ff3d3fef9c72747f4a6b26fed38e/cryptography-50.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367", size = 4697300, upload-time = "2026-08-25T19:44:35.144Z" }, + { url = "https://files.pythonhosted.org/packages/08/bd/ed5396be499ffcf8807a585bfe38b71a1fbdd1c342b4f9b6d0ef5162a946/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5", size = 4716039, upload-time = "2026-08-25T19:44:37.192Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6e/1cf405c5c8e8df7545378048e954792f00b7f2367af8863ce8b8f3e10607/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9", size = 5332388, upload-time = "2026-08-25T19:44:39.16Z" }, + { url = "https://files.pythonhosted.org/packages/47/92/b4317e8c32c4f47b062f5398bd79106b220a124546f42be83bf32b761e2a/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0", size = 4730293, upload-time = "2026-08-25T19:44:41.298Z" }, + { url = "https://files.pythonhosted.org/packages/39/0d/a1e7633e2c744d0f2983320a27e924ef2264c79c56e1a58d5fb0a1cfd413/cryptography-50.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc", size = 4346031, upload-time = "2026-08-25T19:44:43.245Z" }, + { url = "https://files.pythonhosted.org/packages/88/dd/b215616f9bab3fc18510c78a4e5c9f362d77838503c363dc747c7d4f5c6f/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17", size = 4715344, upload-time = "2026-08-25T19:44:45.291Z" }, + { url = "https://files.pythonhosted.org/packages/b1/1b/ec3ebd31741d0e963612c4fe43caa39341b9b1e031e469820e42e4c83918/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6", size = 5287201, upload-time = "2026-08-25T19:44:47.297Z" }, + { url = "https://files.pythonhosted.org/packages/1a/01/0127d11a762b31a9ee0221894f540318761783f3fdc4bc5d057698caebd5/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3", size = 4730023, upload-time = "2026-08-25T19:44:49.435Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b9/e7425ebfb599241a0c1d7000f1b466c3062da66c19d9525031315dff7213/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6", size = 4847362, upload-time = "2026-08-25T19:44:51.94Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fd/60d0ddf4defa12e482c9d5e0f554384d6e8ab25341fd15f060028fd92e6a/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149", size = 4999247, upload-time = "2026-08-25T19:44:53.876Z" }, + { url = "https://files.pythonhosted.org/packages/4d/56/bc4f2b209e766c93372cfcd59b781a0b2b59700f62a969580415b699c2b2/cryptography-50.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf", size = 3825806, upload-time = "2026-08-25T19:44:56.209Z" }, + { url = "https://files.pythonhosted.org/packages/84/a9/ee16a903f13755e914d1eecc482fe64d1f10761c3960e5d8fa6837377aff/cryptography-50.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0", size = 4035307, upload-time = "2026-08-25T19:44:58.305Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a5/9ec7e81e8526c0d7a387d73386b2daed3f39e10d81a85930bd1b6bfba65c/cryptography-50.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23", size = 4751900, upload-time = "2026-08-25T19:45:00.401Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3c/0e77bd5ffcf078e9dd27d3074aad6c030d9b10d0bf69329d573c927a188c/cryptography-50.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733", size = 4738357, upload-time = "2026-08-25T19:45:02.786Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/3c5f80daa4dcd47323c7af8a2fcb90de27a33564d4fcac69846c0972691a/cryptography-50.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88", size = 4758474, upload-time = "2026-08-25T19:45:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2b/214cf0cf93db9628c3c20c896b229f327f6fb1b20e4b3743d8ad3f00af8b/cryptography-50.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054", size = 5375862, upload-time = "2026-08-25T19:45:07.163Z" }, + { url = "https://files.pythonhosted.org/packages/d6/51/3f9701867a46b6c1740c9b52fc4d3bed6cbdcfedcc9b6e64305c07f39cff/cryptography-50.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5", size = 4772942, upload-time = "2026-08-25T19:45:09.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5c/13ea642e08e2544d0f5396122055f4820cfacb3203562197b5967125ea97/cryptography-50.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361", size = 4383347, upload-time = "2026-08-25T19:45:11.659Z" }, + { url = "https://files.pythonhosted.org/packages/84/d5/7d1fe1cb93f91c428093ff234e128c89ba8ea61a6f26aab406081f9b996e/cryptography-50.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71", size = 4758050, upload-time = "2026-08-25T19:45:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/557fc5ead96a829e0bc812a3b9dc4a52a2f27e4f7f5950da7ff27653a805/cryptography-50.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80", size = 5332955, upload-time = "2026-08-25T19:45:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/eb/5d7124083e8d8cda8f5b348f544b71ad6f707ad63193758ef4d8e569da02/cryptography-50.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239", size = 4772694, upload-time = "2026-08-25T19:45:18.315Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/f1f955e0921dd2b6d22eae7e8d24a4c4b638d10735ffbf6a71f99eb0fcb8/cryptography-50.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558", size = 4888413, upload-time = "2026-08-25T19:45:20.4Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ab/89e2b798d2c3925f82e2bb72d5979f3d2f6da2dd22ef4a8cd8b70d920039/cryptography-50.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e", size = 5044355, upload-time = "2026-08-25T19:45:22.353Z" }, + { url = "https://files.pythonhosted.org/packages/99/89/87ef49ffe383ef4e147d27b7bf2088fb0b54ea409dd87b5a89442e5828a5/cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2", size = 3875429, upload-time = "2026-08-25T19:45:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/c7/27/8d207af749c453ee17ea087340b3f2b4adef75aadd1d277b1b129bdda84e/cryptography-50.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94", size = 3974350, upload-time = "2026-08-25T19:45:26.551Z" }, + { url = "https://files.pythonhosted.org/packages/14/9a/6d3a4d7852e22d657438b7bf51f66102c7d71c0e1fafeec652281d0403e5/cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f", size = 4698675, upload-time = "2026-08-25T19:45:28.658Z" }, + { url = "https://files.pythonhosted.org/packages/73/35/5c3717edf9e68a0550ce04e28eab493fe545eccd81742af03f6a75fe260b/cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671", size = 4707410, upload-time = "2026-08-25T19:45:30.816Z" }, + { url = "https://files.pythonhosted.org/packages/1d/e0/e786934472e3ac4ecdecc7b129a0ca1a2a40dffdafcf2c3ea9d4397f8def/cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e", size = 4698378, upload-time = "2026-08-25T19:45:33.043Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/5b3f53a0b74d122f023476ede40ba5d3e70d5cf475f73b899740d26a4fb2/cryptography-50.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6", size = 4706889, upload-time = "2026-08-25T19:45:35.086Z" }, + { url = "https://files.pythonhosted.org/packages/71/44/711e61f7d014be825ef79b285b047292d1bf893732ac1bc030a351fb517f/cryptography-50.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b", size = 3824006, upload-time = "2026-08-25T19:45:37.281Z" }, ] [[package]] @@ -1200,11 +1192,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] [[package]] @@ -1671,7 +1663,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.26.0" +version = "1.30.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1689,9 +1681,9 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005, upload-time = "2026-01-24T19:40:32.468Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/93/0142dc84a666daf8ad51a34268f34c12fd6fda4f3810c4be2504eecc8212/mcp-1.30.0.tar.gz", hash = "sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4", size = 680511, upload-time = "2026-09-07T14:34:15.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/d9/eaa1f80170d2b7c5ba23f3b59f766f3a0bb41155fbc32a69adfa1adaaef9/mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca", size = 233615, upload-time = "2026-01-24T19:40:30.652Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f4/e58bc33317c92a0203664daaf00bf6f41166cc0149e5d6870a03f7cd004a/mcp-1.30.0-py3-none-any.whl", hash = "sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0", size = 234581, upload-time = "2026-09-07T14:34:14.266Z" }, ] [[package]] @@ -4276,11 +4268,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.1" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/1d/0f3a93cca1ac5e8287842ed4eebbd0f7a991315089b1a0b01c7788aa7b63/urllib3-2.6.1.tar.gz", hash = "sha256:5379eb6e1aba4088bae84f8242960017ec8d8e3decf30480b3a1abdaa9671a3f", size = 432678, upload-time = "2025-12-08T15:25:26.773Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/56/190ceb8cb10511b730b564fb1e0293fa468363dbad26145c34928a60cb0c/urllib3-2.6.1-py3-none-any.whl", hash = "sha256:e67d06fe947c36a7ca39f4994b08d73922d40e6cca949907be05efa6fd75110b", size = 131138, upload-time = "2025-12-08T15:25:25.51Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] From 1db1d1e386636a64179f5359d0a55ad293b0a059 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:01:49 -0400 Subject: [PATCH 05/72] test: sweep every v7 tool and resource for workspace isolation and path leaks Pass 1 runs the production server in-process as the loopback principal with two registered workspaces. It seeds A with canary text, records, an outcome, a relationship, a rule, a trigger, active context, cursors and selection tokens, then calls every manifest tool and workspace resource against B with A's IDs, cursors, tokens and idempotency keys. No B response may contain A's text or IDs, A's ledger tables may not change, a rejected call may not change B, and no field outside user text may carry a registered root, a storage path or the home directory. Consolidation and federated recall are refused without a directional link, and a B-side link is what makes consolidation from A possible. Pass 2 relaunches the same workspaces over HTTP with a JWT principal granted only A and requires UNAUTHORIZED_WORKSPACE for every call that targets B, including B as a consolidation or federated-recall source. A fast guard fails when a manifest tool is neither swept nor declared cross-workspace. The JWT issuer and workspace setup move into process_client.py so the access test and the sweep share them. Co-Authored-By: Claude Opus 5 --- tests/api_v7/process_client.py | 94 ++- tests/api_v7/test_process_workspace_access.py | 91 +-- .../test_process_workspace_isolation_sweep.py | 676 ++++++++++++++++++ 3 files changed, 786 insertions(+), 75 deletions(-) create mode 100644 tests/api_v7/test_process_workspace_isolation_sweep.py diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index 75361ac..1f2f795 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -8,16 +8,106 @@ import socket import subprocess import sys -from collections.abc import Awaitable, Callable -from contextlib import asynccontextmanager +import threading +import time +from collections.abc import Awaitable, Callable, Iterator +from contextlib import asynccontextmanager, contextmanager from datetime import timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import httpx +from joserfc import jwt +from joserfc.jwk import RSAKey from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client from mcp.client.streamable_http import streamable_http_client +from daem0nmcp.database import DatabaseManager +from daem0nmcp.protected_files import write_new_owner_only_file +from daem0nmcp.workspace import Workspace, WorkspaceRegistry + +JWT_AUDIENCE = "daem0n-access-test" +JWT_SUBJECT = "alice" + + +async def initialize_workspaces(roots: tuple[Path, ...]) -> list[Workspace]: + """Create an initialized store below each root and return its workspace.""" + workspaces = [] + for root in roots: + storage = root / ".daem0nmcp" / "storage" + storage.mkdir(parents=True) + manager = DatabaseManager(str(storage)) + try: + await manager.init_db() + finally: + await manager.close() + workspaces.append(WorkspaceRegistry([root], default_root=root).default) + return workspaces + + +@contextmanager +def jwt_issuer() -> Iterator[tuple[str, Callable[..., str]]]: + """Serve a JWKS on loopback and yield its issuer URL and a token factory.""" + key = RSAKey.generate_key(2048, parameters={"kid": "workspace-access-test"}) + body = json.dumps({"keys": [key.as_dict(private=False)]}).encode() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + issuer_url = f"http://127.0.0.1:{server.server_port}" + + def token(**overrides) -> str: + claims = { + "iss": issuer_url, + "aud": JWT_AUDIENCE, + "sub": JWT_SUBJECT, + "exp": int(time.time()) + 300, + **overrides, + } + return jwt.encode({"alg": "RS256", "kid": "workspace-access-test"}, claims, key) + + try: + yield issuer_url, token + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + assert not thread.is_alive() + + +def write_workspace_grants(policy_path: Path, workspace_ids: list[str]) -> None: + """Grant the test JWT subject exactly these workspaces.""" + body = json.dumps( + {"schema_version": 1, "grants": {f"oauth-sub:{JWT_SUBJECT}": workspace_ids}} + ).encode() + if policy_path.exists(): + policy_path.write_bytes(body) + else: + write_new_owner_only_file(policy_path, body) + + +def jwt_environment(issuer_url: str, policy_path: Path) -> dict[str, str]: + """Server environment that verifies the test issuer's JWTs.""" + return { + "FASTMCP_SERVER_AUTH": "fastmcp.server.auth.providers.jwt.JWTVerifier", + "FASTMCP_SERVER_AUTH_JWT_JWKS_URI": issuer_url + "/jwks.json", + "FASTMCP_SERVER_AUTH_JWT_ISSUER": issuer_url, + "FASTMCP_SERVER_AUTH_JWT_AUDIENCE": JWT_AUDIENCE, + "DAEM0NMCP_WORKSPACE_ACCESS_FILE": str(policy_path), + } + def server_environment( workspace: Path, diff --git a/tests/api_v7/test_process_workspace_access.py b/tests/api_v7/test_process_workspace_access.py index 66b4dcd..f2687fc 100644 --- a/tests/api_v7/test_process_workspace_access.py +++ b/tests/api_v7/test_process_workspace_access.py @@ -2,92 +2,37 @@ from __future__ import annotations -import json -import threading import time -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import httpx import pytest -from joserfc import jwt -from joserfc.jwk import RSAKey from mcp.shared.exceptions import McpError -from daem0nmcp.database import DatabaseManager -from daem0nmcp.protected_files import write_new_owner_only_file -from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import call, process_client, succeed +from tests.api_v7.process_client import ( + call, + initialize_workspaces, + jwt_environment, + jwt_issuer, + process_client, + succeed, + write_workspace_grants, +) @pytest.fixture def issuer(): - key = RSAKey.generate_key(2048, parameters={"kid": "workspace-access-test"}) - body = json.dumps({"keys": [key.as_dict(private=False)]}).encode() - - class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, *_args): - pass - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - issuer_url = f"http://127.0.0.1:{server.server_port}" - - def token(**overrides): - claims = { - "iss": issuer_url, - "aud": "daem0n-access-test", - "sub": "alice", - "exp": int(time.time()) + 300, - **overrides, - } - return jwt.encode({"alg": "RS256", "kid": "workspace-access-test"}, claims, key) - - try: - yield issuer_url, token - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - assert not thread.is_alive() + with jwt_issuer() as value: + yield value async def test_real_jwt_workspace_access_and_revocation(tmp_path, issuer): roots = (tmp_path / "alpha", tmp_path / "beta") - workspaces = [] - for root in roots: - storage = root / ".daem0nmcp" / "storage" - storage.mkdir(parents=True) - manager = DatabaseManager(str(storage)) - try: - await manager.init_db() - finally: - await manager.close() - workspaces.append(WorkspaceRegistry([root], default_root=root).default) + workspaces = await initialize_workspaces(roots) alpha, beta = workspaces policy_path = tmp_path / "protected" / "access.json" - - def grants(identifiers): - return json.dumps( - {"schema_version": 1, "grants": {"oauth-sub:alice": identifiers}} - ).encode() - - write_new_owner_only_file(policy_path, grants([alpha.workspace_id])) + write_workspace_grants(policy_path, [alpha.workspace_id]) issuer_url, token = issuer - overrides = { - "FASTMCP_SERVER_AUTH": "fastmcp.server.auth.providers.jwt.JWTVerifier", - "FASTMCP_SERVER_AUTH_JWT_JWKS_URI": issuer_url + "/jwks.json", - "FASTMCP_SERVER_AUTH_JWT_ISSUER": issuer_url, - "FASTMCP_SERVER_AUTH_JWT_AUDIENCE": "daem0n-access-test", - "DAEM0NMCP_WORKSPACE_ACCESS_FILE": str(policy_path), - } + overrides = jwt_environment(issuer_url, policy_path) async def probe(url): async with httpx.AsyncClient(timeout=5) as client: @@ -156,10 +101,10 @@ async def link(): denied = await link() assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE" - policy_path.write_bytes(grants([alpha.workspace_id, beta.workspace_id])) + write_workspace_grants(policy_path, [alpha.workspace_id, beta.workspace_id]) await succeed(session, "session_brief", {"workspace_id": beta.workspace_id}) assert (await link())["ok"] - policy_path.write_bytes(grants([alpha.workspace_id])) + write_workspace_grants(policy_path, [alpha.workspace_id]) denied = await call( session, "memory_recall", @@ -170,7 +115,7 @@ async def link(): }, ) assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE" - policy_path.write_bytes(grants([])) + write_workspace_grants(policy_path, []) denied = await call( session, "memory_recall", @@ -181,7 +126,7 @@ async def link(): await session.read_resource( f"memory://workspaces/{alpha.workspace_id}/warnings" ) - policy_path.write_bytes(grants([beta.workspace_id])) + write_workspace_grants(policy_path, [beta.workspace_id]) await succeed(session, "session_brief", {"workspace_id": beta.workspace_id}) await succeed( session, diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py new file mode 100644 index 0000000..520475d --- /dev/null +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -0,0 +1,676 @@ +"""Workspace isolation and host-path leak sweep across every v7 tool and resource. + +Pass 1 drives the real production server in-process as the default loopback +principal, which may use both registered workspaces. Every manifest tool and +workspace resource is called against workspace B with workspace A's IDs, +cursors, tokens and idempotency keys. No B response may carry A's seeded text +or IDs, A's database may not change, a rejected call may not change B, and no +response field outside user text may carry a registered root, a storage path +or the home directory. + +Pass 2 relaunches the same workspaces over HTTP with a JWT principal granted +only A. Every call that targets B, directly or as a consolidation or +federation source, must return UNAUTHORIZED_WORKSPACE. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections.abc import Awaitable, Callable +from contextlib import closing +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client +from mcp.shared.exceptions import McpError + +from daem0nmcp.api.v7.policy import V7_TOOL_LEVELS +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.config import Settings +from daem0nmcp.covenant import CovenantLevel +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import ( + call, + initialize_workspaces, + jwt_environment, + jwt_issuer, + process_client, + succeed, + write_workspace_grants, +) + +CANARY = "canary7f3a9d1e" + +# Tools whose purpose is to read or move data between workspaces. Each is +# gated by a directional link (or a preview token that required one) plus an +# authorization check for every workspace involved; see the cross checks below. +CROSS_WORKSPACE_TOOLS = ( + "workspace_consolidation_preview", + "workspace_consolidate", + "workspace_consolidate_and_archive_sources", + "workspace_link", + "workspace_unlink", +) + +# Fields that echo text the caller wrote. They are exempt from the host-path +# assertion only (KD-2); A's seeded text and IDs are forbidden in every field. +USER_TEXT_FIELDS = frozenset( + { + "content", + "excerpt", + "bounded_excerpt", + "rationale", + "context", + "outcome_text", + "summary", + "text", + "description", + "label", + "topic", + "trigger", + "recall_query", + "proposed_action", + } +) + +# Derived tables that background projection jobs rewrite after a commit, +# including lazily allocated projection-scoped public IDs. +_DERIVED_TABLE_PREFIXES = ( + "retrieval_", + "discovery_", + "dense_", + "memories_fts", + "projection_manifests", + "background_jobs", + "record_outcome_view", + "public_object_ids", + "sqlite_", +) + +Invoke = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] + + +def _placeholder(prefix: str) -> str: + return f"{prefix}_{hashlib.sha256(prefix.encode()).hexdigest()}" + + +def _ledger_counts(root: Path) -> dict[str, int]: + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as db: + tables = [ + name + for (name,) in db.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + if not name.startswith(_DERIVED_TABLE_PREFIXES) + ] + return { + name: db.execute(f'SELECT count(*) FROM "{name}"').fetchone()[0] + for name in tables + } + + +def _strings(value: Any, key: str | None = None): + if isinstance(value, dict): + for child_key, child in value.items(): + yield from _strings(child, child_key) + elif isinstance(value, list): + for child in value: + yield from _strings(child, key) + elif isinstance(value, str): + yield key, value + + +class Seed: + """Everything workspace A owns that B must never see or resolve.""" + + def __init__(self) -> None: + self.texts: list[str] = [] + self.ids: dict[str, str] = {} + self.tokens: dict[str, str] = {} + + def text(self, label: str) -> str: + value = f"{CANARY} {label} owned by workspace A" + self.texts.append(value) + return value + + def id(self, name: str, prefix: str) -> str: + return self.ids.get(name) or _placeholder(prefix) + + def token(self, name: str) -> str: + return self.tokens.get(name) or f"{name}-{CANARY}-placeholder" + + @staticmethod + def key(name: str) -> str: + """A's idempotency keys, replayed against B.""" + return f"sweep-{name}-0001" + + +async def _protected(invoke: Invoke, workspace_id: str, tool: str, arguments: dict): + grant = await invoke( + "memory_preflight", + { + "workspace_id": workspace_id, + "target_tool": tool, + "target_arguments": arguments, + }, + ) + if not grant["ok"]: + return grant + return await invoke( + tool, + { + "workspace_id": workspace_id, + **arguments, + "preflight_token": grant["data"]["preflight_token"], + }, + ) + + +async def _seed_workspace_a(invoke: Invoke, a: str, b: str) -> Seed: + seed = Seed() + + async def ok(tool, arguments, protected=False): + result = await ( + _protected(invoke, a, tool, arguments) + if protected + else invoke(tool, {"workspace_id": a, **arguments}) + ) + assert result["ok"], f"seed {tool}: {result['error']}" + return result["data"] + + async def optional(tool, arguments): + result = await invoke(tool, {"workspace_id": a, **arguments}) + return result["data"] if result["ok"] else None + + await ok("session_brief", {}) + for name, record_type in (("record", "decision"), ("other_record", "warning")): + stored = await ok( + "memory_store", + { + "record_type": record_type, + "content": seed.text(f"{name} content"), + "rationale": seed.text(f"{name} rationale"), + "relative_file_path": "src/owned.py", + "idempotency_key": seed.key(name), + }, + protected=True, + ) + seed.ids[name] = stored["record"]["record_id"] + seed.ids[f"{name}_event"] = stored["event_id"] + outcome = await ok( + "memory_record_outcome", + { + "record_id": seed.ids["record"], + "outcome_text": seed.text("outcome"), + "worked": False, + "idempotency_key": seed.key("outcome"), + }, + ) + seed.ids["outcome_event"] = outcome["outcome_event_id"] + linked = await ok( + "memory_link", + { + "source_record_id": seed.ids["record"], + "target_record_id": seed.ids["other_record"], + "relationship_type": "related_to", + "idempotency_key": seed.key("link"), + }, + protected=True, + ) + seed.ids["relationship"] = next( + value for value in linked["affected_ids"] if value.startswith("rel_") + ) + rule = await ok( + "rule_create", + { + "trigger": seed.text("rule trigger"), + "must_do": [seed.text("rule must do")], + "idempotency_key": seed.key("rule"), + }, + protected=True, + ) + seed.ids["rule"] = rule["rule_id"] + trigger = await ok( + "context_trigger_create", + { + "trigger_type": "file", + "pattern": "src/**/*.py", + "recall_query": seed.text("trigger recall query"), + "idempotency_key": seed.key("trigger"), + }, + protected=True, + ) + seed.ids["trigger"] = trigger["trigger_id"] + active = await ok( + "active_context_add", + {"record_id": seed.ids["record"], "reason": seed.text("active reason")}, + protected=True, + ) + seed.ids["active_context"] = active["active_context_id"] + page = await ok("active_context_list", {}) + seed.tokens["active_context_selection"] = page["selection_token"] + search = await ok("memory_search_text", {"query": CANARY, "limit": 1}) + seed.tokens["cursor"] = search["next_cursor"] + updates = await ok("session_updates_get", {}) + seed.tokens["session_cursor"] = updates["cursor"] + for tool, arguments in ( + ("memory_prune_preview", {}), + ("memory_duplicates_preview", {}), + ("memory_compaction_preview", {"summary": "sweep", "query": CANARY}), + ("dream_duplicates_preview", {}), + ): + data = await optional(tool, arguments) + if data and data.get("selection_token"): + seed.tokens[tool] = data["selection_token"] + export = await ok("workspace_export", {}) + if export.get("export_session_id"): + seed.ids["export_session"] = export["export_session_id"] + # A consolidation preview needs a linked source with at least one record. + assert (await invoke("session_brief", {"workspace_id": b}))["ok"] + b_record = { + "record_type": "decision", + "content": "Workspace B owns this decision.", + "idempotency_key": "sweep-b-own-record-0001", + } + assert (await _protected(invoke, b, "memory_store", b_record))["ok"] + await ok("workspace_link", {"linked_workspace_id": b}, protected=True) + preview = await ok("workspace_consolidation_preview", {"source_workspace_ids": [b]}) + seed.tokens["consolidation"] = preview["selection_token"] + # Optional-profile objects: real IDs when the profile is installed. + code = await optional("code_index", {"relative_root": "src"}) + if code is not None: + found = await optional("code_search", {"query": CANARY}) + if found and found["items"]: + seed.ids["code_entity"] = found["items"][0]["code_entity_id"] + entities = await optional("entity_list", {}) + if entities and entities["items"]: + seed.ids["entity"] = entities["items"][0]["entity_id"] + communities = await optional("community_list", {}) + if communities and communities["items"]: + seed.ids["community"] = communities["items"][0]["community_id"] + return seed + + +def _sweep_arguments(seed: Seed) -> dict[str, dict[str, Any]]: + """Arguments for every non-cross tool, aimed at B but naming A's objects.""" + + record = seed.id("record", "mem") + cursor = seed.token("cursor") + now = datetime.now(timezone.utc).isoformat() + return { + "active_context_add": {"record_id": record, "reason": "sweep"}, + "active_context_clear": { + "selection_token": seed.token("active_context_selection") + }, + "active_context_list": {"cursor": cursor}, + "active_context_remove": { + "active_context_id": seed.id("active_context", "act") + }, + "code_impact_analyze": { + "code_entity_id": seed.id("code_entity", "code"), + }, + "code_index": {"relative_root": "src"}, + "code_refactor_propose": {"relative_file_path": "src/owned.py"}, + "code_search": {"query": CANARY, "cursor": cursor}, + "code_todos_scan": {"relative_root": "src", "cursor": cursor}, + "code_todos_scan_and_store": { + "relative_root": "src", + "idempotency_key": seed.key("record"), + }, + "community_get": {"community_id": seed.id("community", "com")}, + "community_list": { + "parent_community_id": seed.id("community", "com"), + "cursor": cursor, + }, + "community_rebuild": {"idempotency_key": seed.key("record")}, + "context_compress": {"text": "Isolation sweep neutral text. " * 8}, + "context_trigger_create": { + "trigger_type": "file", + "pattern": "src/**/*.py", + "recall_query": "sweep", + "idempotency_key": seed.key("trigger"), + }, + "context_trigger_delete": {"trigger_id": seed.id("trigger", "trg")}, + "context_trigger_list": {"cursor": cursor}, + "context_triggers_match": {"relative_file_path": "src/owned.py"}, + "covenant_status": {}, + "decision_debate": { + "topic": "sweep", + "advocate_position": "sweep for", + "challenger_position": "sweep against", + "idempotency_key": seed.key("record"), + }, + "decision_simulate": {"record_id": record}, + "document_ingest_url": { + "url": "https://127.0.0.1/owned", + "topic": "sweep", + "idempotency_key": seed.key("record"), + }, + "dream_duplicates_preview": {}, + "dream_duplicates_purge": { + "selection_token": seed.token("dream_duplicates_preview") + }, + "edit_preflight": { + "edit_request_id": _placeholder("edt"), + "description": "sweep", + }, + "entity_backfill": {"idempotency_key": seed.key("record")}, + "entity_evolution_trace": {"entity_id": seed.id("entity", "ent")}, + "entity_list": {"cursor": cursor}, + "knowledge_graph_get": {"record_ids": [record], "query": CANARY}, + "knowledge_graph_render": {"record_ids": [record], "query": CANARY}, + "knowledge_graph_stats": {}, + "memory_archive_set": {"record_id": record, "archived": True}, + "memory_at_time_get": {"record_id": record, "valid_time": now}, + "memory_capture_list": {"cursor": cursor}, + "memory_capture_promote": { + "candidate_id": _placeholder("cap"), + "record_type": "decision", + "content": "sweep", + "idempotency_key": seed.key("record"), + }, + "memory_chain_trace": { + "start_record_id": record, + "end_record_id": seed.id("other_record", "mem"), + }, + "memory_compact": { + "summary": "sweep", + "selection_token": seed.token("memory_compaction_preview"), + "idempotency_key": seed.key("record"), + }, + "memory_compaction_preview": {"summary": "sweep", "query": CANARY}, + "memory_duplicates_cleanup": { + "selection_token": seed.token("memory_duplicates_preview") + }, + "memory_duplicates_preview": {}, + "memory_link": { + "source_record_id": record, + "target_record_id": seed.id("other_record", "mem"), + "relationship_type": "related_to", + "idempotency_key": seed.key("link"), + }, + "memory_pin_set": {"record_id": record, "pinned": True}, + "memory_preflight": { + "target_tool": "memory_archive_set", + "target_arguments": {"record_id": record, "archived": True}, + }, + "memory_prune": {"selection_token": seed.token("memory_prune_preview")}, + "memory_prune_preview": {}, + "memory_recall": {"query": CANARY, "record_ids": [record]}, + "memory_recall_entity": {"entity_id": seed.id("entity", "ent")}, + "memory_recall_file": { + "relative_file_path": "src/owned.py", + "cursor": cursor, + }, + "memory_recall_hierarchical": {"query": CANARY}, + "memory_record_outcome": { + "record_id": record, + "outcome_text": "sweep", + "worked": True, + "idempotency_key": seed.key("outcome"), + }, + "memory_related": {"record_id": record}, + "memory_search_text": {"query": CANARY, "cursor": cursor}, + "memory_store": { + "record_type": "decision", + "content": "Workspace B sweep decision.", + "idempotency_key": seed.key("record"), + }, + "memory_store_batch": { + "records": [{"record_type": "learning", "content": "B batch sweep."}], + "idempotency_key": seed.key("other_record"), + }, + "memory_unlink": {"relationship_id": seed.id("relationship", "rel")}, + "memory_verify": {"text": CANARY}, + "memory_versions_list": {"record_id": record, "cursor": cursor}, + "projection_rebuild": {"projection": "lexical"}, + "rule_check": {"proposed_action": CANARY}, + "rule_create": {"trigger": "sweep", "idempotency_key": seed.key("rule")}, + "rule_evolution_analyze": {"rule_id": seed.id("rule", "rule")}, + "rule_list": {"cursor": cursor}, + "rule_update": { + "rule_id": seed.id("rule", "rule"), + "patch": {"enabled": False}, + }, + "sandbox_execute_python": {"code": "print('sweep')"}, + "session_brief": {}, + "session_updates_get": {"after_cursor": seed.token("session_cursor")}, + "system_health": {"include_components": True}, + "workspace_export": { + "export_session_id": seed.id("export_session", "xpt"), + "cursor": cursor, + }, + "workspace_import": { + "import_session_id": seed.id("export_session", "xpt"), + "idempotency_key": seed.key("record"), + }, + "workspace_links_list": {"cursor": cursor}, + } + + +def _cross_arguments(seed: Seed, source: str) -> dict[str, dict[str, Any]]: + token = seed.token("consolidation") + return { + "workspace_consolidation_preview": {"source_workspace_ids": [source]}, + "workspace_consolidate": { + "source_workspace_ids": [source], + "selection_token": token, + "idempotency_key": "sweep-consolidate-0001", + }, + "workspace_consolidate_and_archive_sources": { + "source_workspace_ids": [source], + "selection_token": token, + "idempotency_key": "sweep-consolidate-archive-0001", + }, + "workspace_link": {"linked_workspace_id": source}, + "workspace_unlink": {"linked_workspace_id": source}, + } + + +def _is_protected(tool: str) -> bool: + return V7_TOOL_LEVELS[tool] in {CovenantLevel.COUNSEL, CovenantLevel.DESTRUCTIVE} + + +def test_every_manifest_tool_is_swept_or_declared_cross_workspace(): + swept = set(_sweep_arguments(Seed())) + cross = set(CROSS_WORKSPACE_TOOLS) + assert not swept & cross + assert swept | cross == set(V7_TOOL_LEVELS), "unswept tools: " + ", ".join( + sorted(set(V7_TOOL_LEVELS) - swept - cross) + ) + + +async def test_workspace_isolation_and_path_leak_sweep(tmp_path, capsys): + roots = (tmp_path / "alpha", tmp_path / "beta") + alpha, beta = await initialize_workspaces(roots) + a, b = alpha.workspace_id, beta.workspace_id + source = roots[0] / "src" + source.mkdir() + (source / "owned.py").write_text( + f"def {CANARY}_owned_function():\n return 1 # TODO: {CANARY} owned todo\n", + encoding="utf-8", + ) + settings = Settings( + project_root=str(roots[0]), + workspace_roots=[str(root) for root in roots], + dream_enabled=False, + ) + server = create_v7_server( + "stdio", + settings=settings, + environ={ + "DAEM0NMCP_PROFILE": "core", + "DAEM0NMCP_APPS_ENABLED": "true", + "DAEM0NMCP_GRAPH_ENABLED": "true", + }, + ) + path_needles = { + str(path).lower() + for base in (*roots, Path.home()) + for path in (base, base.resolve()) + } + path_needles |= {str(root / ".daem0nmcp" / "storage").lower() for root in roots} + path_needles |= {Path(needle).as_posix().lower() for needle in path_needles} + + swept: list[str] = [] + async with Client(server) as client: + + async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await client.call_tool(tool, arguments, raise_on_error=False) + assert isinstance(result.structured_content, dict), tool + return result.structured_content + + served = {tool.name for tool in await client.list_tools()} + assert served == set(V7_TOOL_LEVELS) + seed = await _seed_workspace_a(invoke, a, b) + forbidden = [*seed.texts, *seed.ids.values(), a] + a_ledger = _ledger_counts(roots[0]) + + def assert_isolated(label: str, response: Any) -> None: + for key, value in _strings(response): + for secret in forbidden: + assert secret not in value, f"{label}: A's {secret!r} in {key}" + if key in USER_TEXT_FIELDS: + continue + lowered = value.lower() + for needle in path_needles: + assert needle not in lowered, f"{label}: host path in {key}" + assert _ledger_counts(roots[0]) == a_ledger, f"{label} changed A" + + assert (await invoke("session_brief", {"workspace_id": b}))["ok"] + for tool, arguments in sorted(_sweep_arguments(seed).items()): + variants = [arguments] + if {"cursor", "after_cursor"} & set(arguments): + # A's cursor is refused, so also read B's own first page. + variants.append( + { + key: value + for key, value in arguments.items() + if key not in {"cursor", "after_cursor"} + } + ) + for variant in variants: + before = _ledger_counts(roots[1]) + if _is_protected(tool): + response = await _protected(invoke, b, tool, variant) + else: + response = await invoke(tool, {"workspace_id": b, **variant}) + assert_isolated(tool, response) + if not response["ok"]: + assert _ledger_counts(roots[1]) == before, ( + f"rejected {tool} wrote B" + ) + swept.append(tool) + + templates = await client.list_resource_templates() + workspace_templates = [ + template.uriTemplate + for template in templates + if template.uriTemplate.startswith("memory://workspaces/{workspace_id}/") + ] + assert workspace_templates + for template in workspace_templates: + uri = template.replace("{workspace_id}", b) + contents = await client.read_resource(uri) + assert_isolated(uri, [json.loads(content.text) for content in contents]) + swept.append(uri) + + # Cross-workspace tools: B has no link to A (only A -> B exists), so + # every B-side read or move of A must be refused; a federated recall is + # refused the same way. + federated = await invoke( + "memory_recall", + {"workspace_id": b, "query": CANARY, "linked_workspace_ids": [a]}, + ) + assert not federated["ok"] + assert_isolated("memory_recall linked", federated) + for tool, arguments in _cross_arguments(seed, a).items(): + if tool in {"workspace_link", "workspace_unlink"}: + continue + before = _ledger_counts(roots[1]) + if _is_protected(tool): + response = await _protected(invoke, b, tool, arguments) + else: + response = await invoke(tool, {"workspace_id": b, **arguments}) + assert not response["ok"], f"{tool} crossed without a link" + assert_isolated(tool, response) + assert _ledger_counts(roots[1]) == before + # Linking needs a preflight bound to the exact target, then the + # direction it names is what consolidation honours. + unbound = await invoke( + "workspace_link", + { + "workspace_id": b, + "linked_workspace_id": a, + "preflight_token": seed.token("consolidation"), + }, + ) + assert not unbound["ok"] + link = _cross_arguments(seed, a)["workspace_link"] + assert (await _protected(invoke, b, "workspace_link", link))["ok"] + preview = await invoke( + "workspace_consolidation_preview", + {"workspace_id": b, "source_workspace_ids": [a]}, + ) + assert preview["ok"], preview["error"] + unlink = _cross_arguments(seed, a)["workspace_unlink"] + assert (await _protected(invoke, b, "workspace_unlink", unlink))["ok"] + after_unlink = await invoke( + "workspace_consolidation_preview", + {"workspace_id": b, "source_workspace_ids": [a]}, + ) + assert not after_unlink["ok"] + + # Pass 2: a JWT principal granted only A is refused B everywhere. + policy_path = tmp_path / "protected" / "access.json" + write_workspace_grants(policy_path, [a]) + with jwt_issuer() as (issuer_url, token): + async with process_client( + roots[0], + "streamable-http", + workspace_roots=roots, + environment_overrides=jwt_environment(issuer_url, policy_path), + http_headers={"Authorization": "Bearer " + token()}, + ) as session: + + async def remote(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + return await call(session, tool, arguments) + + await succeed(session, "session_brief", {"workspace_id": a}) + placeholder_token = f"preflight-{CANARY}-placeholder" + targets = { + **_sweep_arguments(seed), + **_cross_arguments(seed, a), + } + for tool, arguments in sorted(targets.items()): + if _is_protected(tool): + arguments = {**arguments, "preflight_token": placeholder_token} + denied = await remote(tool, {"workspace_id": b, **arguments}) + assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE", tool + for template in workspace_templates: + with pytest.raises(McpError): + await session.read_resource(template.replace("{workspace_id}", b)) + # B as a source of an A-side federated read or consolidation. + denied = await remote( + "memory_recall", + {"workspace_id": a, "query": CANARY, "linked_workspace_ids": [b]}, + ) + assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE" + for tool, arguments in _cross_arguments(seed, b).items(): + if _is_protected(tool): + denied = await _protected(remote, a, tool, arguments) + else: + denied = await remote(tool, {"workspace_id": a, **arguments}) + assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE", tool + + with capsys.disabled(): + print(f"\nswept ({len(swept)}): {', '.join(swept)}") + print( + f"cross-workspace ({len(CROSS_WORKSPACE_TOOLS)}): " + + ", ".join(CROSS_WORKSPACE_TOOLS) + ) From ebe5e4842ce303be18828cab94f69a9e655a8eb8 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:40:32 -0400 Subject: [PATCH 06/72] fix: let migrate-v7 read v6 databases on Linux and macOS inventory_database opened the source with the unix-none VFS, which has no shared-memory support, so SQLite could not open any WAL database and every migrate-v7 dry run and apply failed on POSIX with 'unable to open database file'. A closed database with no -wal or -shm is now read as immutable (the main file is the whole database and nothing is created); a live WAL is read through the default VFS, whose only write is the volatile -shm wal-index. Windows keeps the win32-none VFS. The dangling-pointer guard test now matches the redacted error code; it was skipped wherever symlinks are unavailable, so it never ran before. Co-Authored-By: Claude Opus 5 --- daem0nmcp/migrations/v7.py | 20 +++++++++++++++----- tests/test_legacy_vector_migration_guards.py | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index 90a2fda..ce778ca 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -146,11 +146,21 @@ def _quoted_identifier(name: str) -> str: def _readonly_connection(path: Path) -> sqlite3.Connection: - # The platform no-lock VFS is essential here: ordinary SQLite read-only WAL - # connections still mutate shared-memory read marks. Migration dry-run is a - # byte-for-byte read-only operation, so it must not touch ``-shm``. - vfs = "win32-none" if sys.platform == "win32" else "unix-none" - uri = "file:" + quote(path.resolve().as_posix(), safe="/:") + f"?mode=ro&vfs={vfs}" + # Migration dry-run must not write any byte. On Windows the no-lock VFS + # keeps read-only connections off the ``-shm`` read marks. POSIX has no + # equivalent: ``unix-none`` lacks shared memory, so it cannot open a WAL + # database at all. There, with no ``-wal``/``-shm`` the main file is the + # whole database and is read as immutable (SQLite would otherwise create + # both files); a live WAL is read through the default VFS, whose only + # write is the volatile ``-shm`` wal-index. + resolved = path.resolve() + if sys.platform == "win32": + parameters = "&vfs=win32-none" + elif not any(Path(f"{resolved}{suffix}").exists() for suffix in ("-wal", "-shm")): + parameters = "&immutable=1" + else: + parameters = "" + uri = "file:" + quote(resolved.as_posix(), safe="/:") + f"?mode=ro{parameters}" connection = sqlite3.connect(uri, uri=True) connection.row_factory = sqlite3.Row connection.execute("PRAGMA query_only=ON") diff --git a/tests/test_legacy_vector_migration_guards.py b/tests/test_legacy_vector_migration_guards.py index 891fce8..f22ac5b 100644 --- a/tests/test_legacy_vector_migration_guards.py +++ b/tests/test_legacy_vector_migration_guards.py @@ -740,7 +740,7 @@ def test_dangling_active_pointer_never_falls_back_to_v6_database(self): except (OSError, NotImplementedError): self.skipTest("symlinks unavailable in this environment") - with self.assertRaisesRegex(ValueError, "active-db"): + with self.assertRaisesRegex(ValueError, "UNSAFE_ACTIVE_POINTER"): upgrade.resolve_db_path(str(root)) self.assertEqual(_OLD_VECTOR, _read_embedding(database)) From fef1ba102dee07eec701b549381eec0afe9b8d6c Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:40:32 -0400 Subject: [PATCH 07/72] fix: make fresh bootstrap and graph builds work on macOS Fresh-workspace bootstrap staged its database under the system temp directory, which on macOS sits below the /var -> /private/var (or /tmp) symlink, so the linked-ancestry guard rejected it and every new workspace reported ACTIVE_V7_UNAVAILABLE. Resolve the temp root first. The Leiden worker capped itself with setrlimit(RLIMIT_AS), which macOS rejects because a process already reserves more address space than the cap, so every graph build ended CAPABILITY_DEGRADED. Skip the cap on macOS as on Windows; the deadline and request bounds still apply. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/workspace_bootstrap.py | 8 +++++++- daem0nmcp/graph/leiden.py | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/daem0nmcp/api/v7/workspace_bootstrap.py b/daem0nmcp/api/v7/workspace_bootstrap.py index c4c93c7..213ba63 100644 --- a/daem0nmcp/api/v7/workspace_bootstrap.py +++ b/daem0nmcp/api/v7/workspace_bootstrap.py @@ -498,7 +498,13 @@ async def _bootstrap_posix_staged( ) -> None: """Build privately, then publish only through the retained directory FD.""" - with tempfile.TemporaryDirectory(prefix="daem0nmcp-v7-bootstrap-") as raw: + # The system temp root is resolved first: on macOS it lives below the + # /var -> /private/var (or /tmp -> /private/tmp) symlink, which the + # linked-ancestry guard on the staging lock would otherwise reject. + with tempfile.TemporaryDirectory( + prefix="daem0nmcp-v7-bootstrap-", + dir=Path(tempfile.gettempdir()).resolve(strict=True), + ) as raw: staging = Path(raw) / "storage" manager = await _construct_staging_manager(staging, workspace.root) try: diff --git a/daem0nmcp/graph/leiden.py b/daem0nmcp/graph/leiden.py index 3c7ffe6..496cee3 100644 --- a/daem0nmcp/graph/leiden.py +++ b/daem0nmcp/graph/leiden.py @@ -361,7 +361,11 @@ def run_leiden_bounded( def _apply_self_memory_limit(limit_bytes: int) -> None: - if sys.platform == "win32": + # Windows bounds the worker with a job object instead. macOS reserves far + # more address space than any sane cap at startup, so setrlimit(RLIMIT_AS) + # is rejected there and cannot bound the worker; the deadline and the + # request-size limits still apply. + if sys.platform in {"win32", "darwin"}: return try: import resource From b881ca606d3bb7f0096a87c46698bf1c671e15e8 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:40:32 -0400 Subject: [PATCH 08/72] fix: keep local edit-bridge sockets under the AF_UNIX path limit The default socket path was ~/.daem0nmcp/edit-bridges/<64 hex>/run/ bridge-<24 hex>.sock, about 140 bytes, beyond the 104-108 byte AF_UNIX limit, so the local edit bridge could never start on Linux or macOS. Sockets now live in /run/<16 hex>/b-<24 hex>.sock, and a test bounds the suffix. Tests that open sockets use a short socket_tmp_path because the repository-local tmp_path is too deep on CI runners. A backslash in a POSIX edit path now raises NativeEditNormalizationError instead of leaking a ValueError; the apply_patch test uses a backslash separator only on Windows, where OpenCode emits one. Co-Authored-By: Claude Opus 5 --- daem0nmcp/claude_hooks/native_edit.py | 4 +++- daem0nmcp/edit_bridge_transport.py | 2 +- daem0nmcp/edit_host.py | 5 ++++- tests/api_v7/test_process_edit_capture.py | 3 ++- tests/conftest.py | 17 +++++++++++++++++ tests/test_claude_native_edit.py | 18 ++++++++++++++++-- tests/test_edit_bridge_transport.py | 14 ++++++++------ tests/test_edit_host.py | 10 ++++++++++ 8 files changed, 61 insertions(+), 12 deletions(-) diff --git a/daem0nmcp/claude_hooks/native_edit.py b/daem0nmcp/claude_hooks/native_edit.py index 36d2532..279cf96 100644 --- a/daem0nmcp/claude_hooks/native_edit.py +++ b/daem0nmcp/claude_hooks/native_edit.py @@ -151,7 +151,9 @@ def _relative_path(root: Path, value: object) -> str: raise NativeEditNormalizationError( "native edit path is outside workspace" ) from exc - if not relative or relative == "." or len(relative) > 1024: + # A backslash is a separator only on Windows (as_posix() converts it + # there); on POSIX it is a filename character the bridge cannot represent. + if not relative or relative == "." or len(relative) > 1024 or "\\" in relative: raise NativeEditNormalizationError("native edit path is invalid") return relative diff --git a/daem0nmcp/edit_bridge_transport.py b/daem0nmcp/edit_bridge_transport.py index 4a9e758..09d2da5 100644 --- a/daem0nmcp/edit_bridge_transport.py +++ b/daem0nmcp/edit_bridge_transport.py @@ -576,7 +576,7 @@ def local_bridge_address(runtime_directory: Path, credential_id: str) -> str: if sys.platform == "win32": return rf"\\.\pipe\daem0nmcp-{digest}" directory = ensure_owner_only_directory(runtime_directory) - return str(directory / f"bridge-{digest}.sock") + return str(directory / f"b-{digest}.sock") class LocalBridgeServer: diff --git a/daem0nmcp/edit_host.py b/daem0nmcp/edit_host.py index 161d9c0..2bb3d08 100644 --- a/daem0nmcp/edit_host.py +++ b/daem0nmcp/edit_host.py @@ -610,7 +610,10 @@ def provision_local_bridge_installation( reject_linked_ancestry(base) authority_directory = ensure_owner_only_directory(base.absolute() / authority_id) credential_path = authority_directory / "credential.json" - runtime_directory = authority_directory / "run" + # AF_UNIX socket paths are limited to 104-108 bytes, and the authority + # directory name alone is 64 characters, so sockets live in a short + # sibling directory instead. + runtime_directory = base.absolute() / "run" / authority_id[:16] created = False if credential_path.exists(): _, identity = load_bridge_credential(credential_path) diff --git a/tests/api_v7/test_process_edit_capture.py b/tests/api_v7/test_process_edit_capture.py index 7534bf4..6118ab9 100644 --- a/tests/api_v7/test_process_edit_capture.py +++ b/tests/api_v7/test_process_edit_capture.py @@ -37,11 +37,12 @@ async def initialized_workspace(tmp_path): async def test_actual_stdio_mcp_shares_bridge_authority_and_promotes_capture( initialized_workspace, + socket_tmp_path, ): workspace = initialized_workspace installation = provision_local_bridge_installation( workspace.root, - config_root=workspace.root.parent / f"{workspace.root.name}-host", + config_root=socket_tmp_path / "host", ) environment = installation.environment() scope = {"workspace_id": workspace.workspace_id} diff --git a/tests/conftest.py b/tests/conftest.py index 9a7df7d..5b67fd7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import getpass import os import shutil +import sys import tempfile import uuid from pathlib import Path @@ -70,6 +71,22 @@ def tmp_path(tmp_path_factory): shutil.rmtree(path, ignore_errors=True) +@pytest.fixture +def socket_tmp_path(tmp_path): + """A directory short enough to hold AF_UNIX sockets (104-108 byte paths). + + The repository-local tmp_path is too deep on CI runners; Windows uses named + pipes, so it keeps the ordinary tmp_path. + """ + if sys.platform == "win32": + yield tmp_path + return + path = Path("/tmp").resolve() / f"d7-{uuid.uuid4().hex[:12]}" + path.mkdir(mode=0o700) + yield path + shutil.rmtree(path, ignore_errors=True) + + @pytest.fixture def covenant_workspace_factory(): """Create explicit isolated Covenant scopes for test workspaces.""" diff --git a/tests/test_claude_native_edit.py b/tests/test_claude_native_edit.py index 2821ab5..ca02c48 100644 --- a/tests/test_claude_native_edit.py +++ b/tests/test_claude_native_edit.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import sys import pytest @@ -106,8 +107,10 @@ def test_opencode_apply_patch_binds_raw_arguments_and_all_path_preimages(tmp_pat (tmp_path / "src" / "update.py").write_text("before\n", encoding="utf-8") (tmp_path / "src" / "delete.py").write_text("delete\n", encoding="utf-8") (tmp_path / "src" / "move.py").write_text("move\n", encoding="utf-8") - patch_text = """*** Begin Patch -*** Update File: src\\update.py + # OpenCode on Windows may emit backslash separators; POSIX never does. + separator = "\\" if sys.platform == "win32" else "/" + patch_text = f"""*** Begin Patch +*** Update File: src{separator}update.py @@ -before +after @@ -144,6 +147,17 @@ def test_opencode_apply_patch_binds_raw_arguments_and_all_path_preimages(tmp_pat ] +@pytest.mark.skipif(sys.platform == "win32", reason="backslash is a separator") +def test_posix_backslash_filename_is_a_normalization_error(tmp_path): + with pytest.raises(NativeEditNormalizationError): + native_edit_request( + project_path=tmp_path, + tool_name="Write", + tool_input={"file_path": "src\\new.py", "content": "x"}, + configured_tools=frozenset({"Write"}), + ) + + def test_opencode_apply_patch_post_paths_do_not_recheck_changed_state(tmp_path): patch_text = """*** Begin Patch *** Add File: added.txt diff --git a/tests/test_edit_bridge_transport.py b/tests/test_edit_bridge_transport.py index 35ec434..15c0e77 100644 --- a/tests/test_edit_bridge_transport.py +++ b/tests/test_edit_bridge_transport.py @@ -154,10 +154,10 @@ async def test_every_scoped_route_rechecks_workspace_authority( @pytest.mark.asyncio async def test_local_ipc_requires_transport_auth_and_consumes_exact_retry( - tmp_path, transport_context + socket_tmp_path, transport_context ): workspace, identity, broker, protocol = transport_context - address = local_bridge_address(tmp_path / "run", identity.credential_id) + address = local_bridge_address(socket_tmp_path / "run", identity.credential_id) server = LocalBridgeServer(protocol=protocol, address=address, authkey=AUTHKEY) server.start() client = LocalBridgeClient(address=address, authkey=AUTHKEY, bearer=SECRET) @@ -300,10 +300,12 @@ def _local_raw_client(address): @pytest.mark.asyncio async def test_local_ipc_bounds_idle_bad_auth_duplicate_json_and_shutdown( - tmp_path, transport_context + socket_tmp_path, transport_context ): workspace, identity, _, protocol = transport_context - address = local_bridge_address(tmp_path / "adversarial", identity.credential_id) + address = local_bridge_address( + socket_tmp_path / "adversarial", identity.credential_id + ) server = LocalBridgeServer(protocol=protocol, address=address, authkey=AUTHKEY) server.start() client = LocalBridgeClient( @@ -354,8 +356,8 @@ async def test_local_ipc_bounds_idle_bad_auth_duplicate_json_and_shutdown( assert time.monotonic() - shutdown_started < 4 -def test_local_client_deadline_interrupts_a_stalled_server(tmp_path) -> None: - address = local_bridge_address(tmp_path / "stalled", "credential-stalled") +def test_local_client_deadline_interrupts_a_stalled_server(socket_tmp_path) -> None: + address = local_bridge_address(socket_tmp_path / "stalled", "credential-stalled") family = "AF_PIPE" if sys.platform == "win32" else "AF_UNIX" listener = multiprocessing.connection.Listener(address, family=family) release = threading.Event() diff --git a/tests/test_edit_host.py b/tests/test_edit_host.py index 41d9c16..8a5e2a0 100644 --- a/tests/test_edit_host.py +++ b/tests/test_edit_host.py @@ -7,12 +7,14 @@ import subprocess import sys from datetime import datetime, timedelta, timezone +from pathlib import Path import pytest from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.edit_bridge_transport import ( RemoteBridgeHTTPSClient, + local_bridge_address, provision_bridge_credential, ) from daem0nmcp.edit_host import ( @@ -59,6 +61,14 @@ def test_local_installation_reuses_host_only_authority_credential(tmp_path): config = EditHostConfig.from_environment(first.environment()) assert config.identity.principal_id == first.principal_id assert config.runtime_directory == first.runtime_directory + # The default root is ~/.daem0nmcp/edit-bridges; with this suffix the + # socket fits macOS's 104-byte AF_UNIX limit for usernames up to 17 chars. + address = local_bridge_address( + first.runtime_directory, config.identity.credential_id + ) + if sys.platform != "win32": + suffix = Path(address).relative_to(host_root.resolve()) + assert len(str(suffix)) <= 52, suffix assert ( config.workspace_id(project) == WorkspaceRegistry(default_root=project).default.workspace_id From 8b03dfbe92e812f441bbf1372f2c3ce4ea53b388 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:40:32 -0400 Subject: [PATCH 09/72] fix: take ownership of protected files created by elevated administrators An elevated administrator's new files are owned by BUILTIN\Administrators, so the owner-only ACL check always failed for them (and on the Windows CI runners). After applying the owner-only DACL, which grants this user WRITE_OWNER, set the owner to the current user when it differs. The verification is unchanged. Co-Authored-By: Claude Opus 5 --- daem0nmcp/protected_files.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/daem0nmcp/protected_files.py b/daem0nmcp/protected_files.py index 0392059..c5fdce9 100644 --- a/daem0nmcp/protected_files.py +++ b/daem0nmcp/protected_files.py @@ -263,6 +263,16 @@ def _protect_windows(path: Path, *, directory: bool) -> None: str(path), _SECURITY_INFORMATION, descriptor ): raise _win_error("setting protected path security") + # An elevated administrator's new files are owned by + # BUILTIN\Administrators. The owner-only DACL just applied grants + # this user WRITE_OWNER, so take ownership explicitly. + owner = expected.split("D:", 1)[0] + "D:" + if not _actual_sddl(path).startswith( + owner + ) and not _ADVAPI32.SetFileSecurityW( + str(path), _OWNER_SECURITY_INFORMATION, descriptor + ): + raise _win_error("setting protected path owner") finally: _KERNEL32.LocalFree(descriptor) if _actual_sddl(path) != expected: From a8ac946b2b3bfa5ef9c217b0bc57d74342fd1d77 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:40:32 -0400 Subject: [PATCH 10/72] test: let the cancelled consolidation observe cancellation before resuming The test cancelled the task and released the blocked worker in the same step, so a fast worker could record the next archive before the task saw its cancellation. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_consolidation_operations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/api_v7/test_consolidation_operations.py b/tests/api_v7/test_consolidation_operations.py index 93ba422..d6af737 100644 --- a/tests/api_v7/test_consolidation_operations.py +++ b/tests/api_v7/test_consolidation_operations.py @@ -339,6 +339,8 @@ def blocked_root(connection, workspace_id): ) self.assertTrue(await asyncio.to_thread(entered.wait, 5)) task.cancel() + # Let the task observe its cancellation before the worker resumes. + await asyncio.sleep(0) release.set() with self.assertRaises(asyncio.CancelledError): await task @@ -576,6 +578,8 @@ def block_first_archive(store, command): ) self.assertTrue(await asyncio.to_thread(entered.wait, 5)) task.cancel() + # Let the task observe its cancellation before the worker resumes. + await asyncio.sleep(0) release.set() with self.assertRaises(asyncio.CancelledError): await task From 9a40570e66168c8e89e6be41b26bca10ec21de33 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:41:28 -0400 Subject: [PATCH 11/72] ci: pin ruff and mypy in the dev extra, CI and pre-commit The dev extra now pins ruff 0.16.8 and mypy 2.3.1 (locked), CI installs the same mypy, and the pre-commit mypy mirror matches it, so following AGENTS.md reproduces the lint and type-check gates. AGENTS.md also notes the platforms the locked cryptography ships wheels for. The stdio duplicate-key docstring no longer names a specific MCP SDK version. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 4 +- .pre-commit-config.yaml | 2 +- AGENTS.md | 6 +- daem0nmcp/transport_security.py | 2 +- pyproject.toml | 3 +- uv.lock | 298 +++++++++++++++++++++++++++++++- 6 files changed, 301 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a5ad75..894ca0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ruff==0.16.8 # keep in sync with uv.lock and .pre-commit-config.yaml + pip install ruff==0.16.8 # keep in sync with pyproject.toml dev extra and .pre-commit-config.yaml - name: Lint with ruff run: | @@ -96,7 +96,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" - pip install mypy + pip install mypy==2.3.1 # keep in sync with pyproject.toml dev extra - name: Type check v7 production surface run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 773d3d1..c41d395 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: # Type checking (non-blocking) - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.0 + rev: v2.3.1 hooks: - id: mypy args: [--ignore-missing-imports] diff --git a/AGENTS.md b/AGENTS.md index 4c90e08..7396711 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,15 +9,15 @@ - Runtime data lives under `.daem0nmcp/` (e.g., `.daem0nmcp/storage/daem0nmcp.db`); do not commit it. ## Build, Test, and Development Commands -- `pip install -e ".[dev,apps,graph]"` installs the package in editable mode with the extras CI tests against. +- `pip install -e ".[dev,apps,graph]"` installs the package in editable mode with the extras CI tests against; the `dev` extra pins the ruff and mypy versions CI uses. (The locked `cryptography` ships wheels only for 64-bit Windows, Apple-silicon macOS and Linux; elsewhere `uv sync --frozen` builds it from source and needs a Rust toolchain.) - `python -m daem0nmcp.server` runs the MCP server directly. - `python start_server.py --port 9876` starts the Windows HTTP launcher. - `python -m daem0nmcp.cli ` runs CLI tasks (example: `python -m daem0nmcp.cli index`). CI (`.github/workflows/ci.yml`) blocks merges on these gates; run them before pushing: - Tests: `pytest tests/ -v --asyncio-mode=auto` (Ubuntu, Windows and macOS on Python 3.10-3.12). -- Lint and format: `ruff check daem0nmcp/ tests/` and `ruff format --check daem0nmcp/ tests/`, with ruff pinned to the `uv.lock` version (0.16.8). -- Type check: `mypy daem0nmcp/api/v7 --ignore-missing-imports --follow-imports=silent` (CI runs it on Linux; add `--platform linux` locally on Windows or macOS). +- Lint and format: `ruff check daem0nmcp/ tests/` and `ruff format --check daem0nmcp/ tests/`, with ruff 0.16.8 (pinned in the `dev` extra). +- Type check (mypy 2.3.1, pinned in the `dev` extra): `mypy daem0nmcp/api/v7 --ignore-missing-imports --follow-imports=silent` (CI runs it on Linux; add `--platform linux` locally on Windows or macOS). - Release inventory: `python scripts/v7_release_inventory.py --check`. ## Coding Style & Naming Conventions diff --git a/daem0nmcp/transport_security.py b/daem0nmcp/transport_security.py index 9a004cd..26a2980 100644 --- a/daem0nmcp/transport_security.py +++ b/daem0nmcp/transport_security.py @@ -330,7 +330,7 @@ def _strict_json_value(raw: str | bytes | bytearray, *, max_bytes: int) -> Any: def strict_stdio_json_boundary(message_model: Any) -> Iterator[None]: """Make the pinned SDK's stdio model parser duplicate-key strict. - MCP SDK 1.26 invokes ``JSONRPCMessage.model_validate_json`` directly for + The pinned MCP SDK invokes ``JSONRPCMessage.model_validate_json`` directly for every line. The Pydantic parser otherwise accepts the last duplicate key, so the reviewed stdio launcher temporarily replaces that one parse seam for the lifetime of the single stdio server. diff --git a/pyproject.toml b/pyproject.toml index e60eed9..09a50ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,8 @@ observability = [ dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.23.0", - "ruff>=0.9.0", + "ruff==0.16.8", # keep in sync with ci.yml and .pre-commit-config.yaml + "mypy==2.3.1", # keep in sync with ci.yml and .pre-commit-config.yaml "pre-commit>=3.0.0", ] diff --git a/uv.lock b/uv.lock index 1236282..6de1c86 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.10" resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -47,7 +48,8 @@ name = "aiofile" version = "3.12.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -104,6 +106,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362, upload-time = "2025-11-28T23:36:57.897Z" }, ] +[[package]] +name = "ast-serialize" +version = "0.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/1e/4f6082cdd6e5a29093513e9a3eabc5ed1c5331a9a84386b2fece80a00a48/ast_serialize-0.11.2.tar.gz", hash = "sha256:976a5bd75845d22f4b52905ddf53ab669ef1b14dba7735f5512841a2ef2b5450", size = 954387, upload-time = "2026-09-13T18:48:55.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/2e/beec3364eef4b01793a676d8cd16e9014c42044a5505000ceae3955e33fa/ast_serialize-0.11.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f6a8dfc5ab204a706f6e5d39c6f77c18c27ef084fa2081803a64a9160ce89277", size = 897089, upload-time = "2026-09-13T18:47:22.69Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d7/ef56443df2891c6ba2c4019c2cb3dcaf97c9948da6d963068e04e8dac6ea/ast_serialize-0.11.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:cb073bfa15742699d408ac50f60878383b5665ae1791d1b6799ea6f08633cd77", size = 1235218, upload-time = "2026-09-13T18:47:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/42/8d/cff58d17ba1d0272ff0b7ab5d3bdfcf8f47317eb0f47c001d394bffebf95/ast_serialize-0.11.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1d6ad94edbe93bf1dabc06c9f37d55b898fdabc456aa6d7ced5e23c14f795f32", size = 1216399, upload-time = "2026-09-13T18:47:26.202Z" }, + { url = "https://files.pythonhosted.org/packages/de/d2/a1da7675af5f42335c36e4da6d86ef4fd7168cead18de81df0a2d6faeb1a/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40b2801cf2221bd922d9f69d2f0ebc373c3db47207315d525b2d87fa161a2af4", size = 1282064, upload-time = "2026-09-13T18:47:27.787Z" }, + { url = "https://files.pythonhosted.org/packages/97/89/5a400a13b2c9c0152ebb5ad45408a3fe5e4e60e325d3ac4e5cf6e915a0cc/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd666cebd6ab3b3c0fd348a6202c26e18a401ee34293c3804d3472266bc146f6", size = 1285864, upload-time = "2026-09-13T18:47:29.667Z" }, + { url = "https://files.pythonhosted.org/packages/02/b8/80a381c70fd49f0316fb0383c4f9e4c13e81b010b64889bd45898ce8f5f4/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0d01f61352c96370febf6c0dbd488dee9183a731fb2702170da9163ae317cded", size = 1554755, upload-time = "2026-09-13T18:47:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/90/97/dcaa34a32d2db789221c125b3eb10feb5089715fe53d9874d627afc26231/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0fd40c668b0fa19b8fdb61d9e63d547e2e19cfbfe053a51ef0b6c37070298a8", size = 1301807, upload-time = "2026-09-13T18:47:32.714Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/84a22420cb312642d7d31547c644d09a3d101418c6d6b9ef2ec30735cf11/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efa819d7c14c8e4153dcd84671826331538be7cbe460383fc6386f5eea5bd234", size = 1301941, upload-time = "2026-09-13T18:47:34.418Z" }, + { url = "https://files.pythonhosted.org/packages/20/8a/aa5f3dcf1aed9678c25982f40d366004e3c0cac47bc0c240f6b837dcbb1f/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a9ffa8a197a721f07a352d0be6185f5b3e6f9aaebfdb66169ed652108531ae3b", size = 1307910, upload-time = "2026-09-13T18:47:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/78/79/91a5102797fe3dc992171382d8579bcb33cbd1424b864ad3117ac43fb3fe/ast_serialize-0.11.2-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00119a8fb8c1dc0f1fab023f4d8071fa49e3b0208ee54d589fd463c16ab0124e", size = 1356258, upload-time = "2026-09-13T18:47:37.984Z" }, + { url = "https://files.pythonhosted.org/packages/51/52/54eeef9918e187ced417c4363eecea66975314cd5b9c91759eef7f7b714b/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0de02520c11391a026e62987a9aa2c3c2ff01545155059ddf0c4bdf2c5ecbe9f", size = 1459057, upload-time = "2026-09-13T18:47:39.891Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/fd84b52b15d42f2423319cffd1fb7f1e9df5d5198e69ab0b449c450254cf/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4e4558956b6a0fb35e18fba58f7d1810b1f2c0e6b52352572cd5dfb6b4ef33a", size = 1562447, upload-time = "2026-09-13T18:47:41.727Z" }, + { url = "https://files.pythonhosted.org/packages/be/92/9fb34f2e64b84a63cca92fb86bd0847b995a63b67477f44c20502fb60352/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6061a54f39e82a9f2cbcb9c268fc441890e4818a6636473caa4f4063254e0750", size = 1556423, upload-time = "2026-09-13T18:47:43.357Z" }, + { url = "https://files.pythonhosted.org/packages/75/0f/c43c44449e7ebc4e83ebd48750088fb06234622faa2d62d2a6dc8970d2a3/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:85fbb01e83967a126d71f679f2b9528ef0912cb0854aa1a4657314c34e255b57", size = 1687156, upload-time = "2026-09-13T18:47:44.995Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a7/9e520f4a79b639da9ee20c1e747c3d739329e902fc55ac38065f25419f56/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:7aaaffc32905159774a107d3cf33dad59bd41b7a0d1bc9885532186753ee7439", size = 1481008, upload-time = "2026-09-13T18:47:46.602Z" }, + { url = "https://files.pythonhosted.org/packages/48/a8/bdd3989f19de09cffcd8179c131f6741a5a8619705fc75b09541ff61530b/ast_serialize-0.11.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:08eda88a0f290a36c38cab33df8bf7e35eb95bc802ca5beb2c8fcda471a7d10c", size = 1501597, upload-time = "2026-09-13T18:47:48.265Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8a/ca2dce2950875a4ef1d7c298196f803b0adcdb7c15ed0cecc71d84bccd70/ast_serialize-0.11.2-cp314-cp314t-win32.whl", hash = "sha256:76cc294246e60a914326b4ca88c6a5ea89c064906614aaf1537ce82f09e9449f", size = 1119503, upload-time = "2026-09-13T18:47:49.896Z" }, + { url = "https://files.pythonhosted.org/packages/5a/12/3f38e3613d07c46f9f81c5b1352748c6552397cc52825502e2c6ae44c6ea/ast_serialize-0.11.2-cp314-cp314t-win_amd64.whl", hash = "sha256:43b51e6ebe6549bf21416c3c78ee886147b80875a87cc6f69e303dde0d75be0b", size = 1156828, upload-time = "2026-09-13T18:47:51.454Z" }, + { url = "https://files.pythonhosted.org/packages/22/90/f89a4f67428a261daafdb69a0d0132c27933268702d1ba47e0b61c51aff1/ast_serialize-0.11.2-cp314-cp314t-win_arm64.whl", hash = "sha256:8df32ad4ff7843734a6c2f067ee974f6d3109ee5a2c3e1a9d2f79347bd282a9a", size = 1128298, upload-time = "2026-09-13T18:47:53.008Z" }, + { url = "https://files.pythonhosted.org/packages/0b/55/a1962188abf0e62d84d55892bb044347e434711763b9a1d4ad867a70c1be/ast_serialize-0.11.2-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:ab924ba260efd7509492f272d4e236d24564033f20c005d7c63a107c6a76fc85", size = 1235457, upload-time = "2026-09-13T18:47:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ad/439c2959150718446af76fbe2f4000f35eba9869ef8564f3d9a3d0b1c370/ast_serialize-0.11.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:a586be418eb70a9f1396cea29ddac8f4b9bf277fb73ea2340db31e218bc00f32", size = 1215705, upload-time = "2026-09-13T18:47:56.178Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d8/2c6542fc3e7c56a0a25d8d12d034d5a2d2e1900e292567b1c1dca8e83124/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8532f20916fa3189d4d785ef2a62d93c4d651ec9c5bffda66d2fc36898351f34", size = 1282530, upload-time = "2026-09-13T18:47:57.619Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ad/6f6755cd0842db46c3b10b1e4735f14aad78d71dea4753eb46933101711b/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee732ae167e686d1d3c00f98d7d82b23138304694f0441b14d7ddf9c0f8a921c", size = 1287792, upload-time = "2026-09-13T18:47:59.227Z" }, + { url = "https://files.pythonhosted.org/packages/03/40/5da672f5dd23fb7dc0c884c97711e56a3540f2fe3c4355a81f8beb385911/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75a1c7f46b9c19fc0ae01ca6fd076301628faa2ed7a8edbd55c6353c483946a3", size = 1557971, upload-time = "2026-09-13T18:48:00.96Z" }, + { url = "https://files.pythonhosted.org/packages/df/c7/2bb25684f697801eb72866fdb94ed5edbff3867ce878b0e542a4a5b9dab9/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2fdf31a0bb85ea2575cc91669f005e6647d2efed491231c4dc1497bc9a5b3aa6", size = 1303230, upload-time = "2026-09-13T18:48:02.337Z" }, + { url = "https://files.pythonhosted.org/packages/d8/85/754681846f26e0ff1da729b1ffe3171e93c22f0aa6ec3cea5b14e3703846/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b78e6fdef3b06c86ed263e1962fee5a7b9d2d158e738b212d13b2c605ee12f5", size = 1302271, upload-time = "2026-09-13T18:48:03.915Z" }, + { url = "https://files.pythonhosted.org/packages/fb/dc/f5521d8cb44b69095c3982ae3658a12c403e0efa19e51aeb9c8a79dff60c/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:8d62a47714c8bc432b9fabcc29989c815c5da17327d35151f2fd0d85c2a7a5ff", size = 1309529, upload-time = "2026-09-13T18:48:05.562Z" }, + { url = "https://files.pythonhosted.org/packages/73/0d/649182c7fd7c4f782279bed514de2dd67e48a5afecb605a098d64fdc01fd/ast_serialize-0.11.2-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8a5ffa70e76191dcf240d3c43e20c93b3bfd26f54d89148c762d57837f5bcd2c", size = 1356869, upload-time = "2026-09-13T18:48:07.534Z" }, + { url = "https://files.pythonhosted.org/packages/bf/cc/aff4d84c16afa742d13a75384127c7d24594dc8c304f0558a15924fd51af/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:bfbe47a3a7c368f28836e78b2440a3643ac0ec4c67d9fe53588e1448f0a3d35d", size = 1460006, upload-time = "2026-09-13T18:48:09.162Z" }, + { url = "https://files.pythonhosted.org/packages/94/a7/891cbec2e5e0d7159196159d3ff0646622f3120ff4576c839ac2dd56c719/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:7f1823275b246f9c7d373be6879e4eec09686948895d4ad083f4b27fd7e4da70", size = 1562935, upload-time = "2026-09-13T18:48:10.978Z" }, + { url = "https://files.pythonhosted.org/packages/45/c4/2c8c4498340ea9aff87a9fd408309aa25d56dd51d7bbddfdb46a3c31424a/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:57c0f5cb0021a5beb1e5e4d6e840ae2f23a28909703ef4d256a144cc1ad3d437", size = 1557109, upload-time = "2026-09-13T18:48:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/0d/8b/c5d4e5226fa18885fe17f949aee3ab1aeb8389c384d946ec1b7c9489cc94/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:cd320a5c4f1f2742af97eea22954f776379175c5ef2504801e9a155f2ff9a4d7", size = 1691603, upload-time = "2026-09-13T18:48:14.293Z" }, + { url = "https://files.pythonhosted.org/packages/73/d6/1d2ca472586f9e3416a289a22f36eeb6dd6f47d77b1a4aba358405babbc7/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:13b13afe32e845c86a573497729e1b7ddeb26c572c78bf50ece51da23b8fad5e", size = 1483053, upload-time = "2026-09-13T18:48:15.789Z" }, + { url = "https://files.pythonhosted.org/packages/0e/16/d3703a7c1e3c76b144ac9349a54d3926d0749918a8dc13a66cede208b8ec/ast_serialize-0.11.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:9d80a81ec84660422579bdb8e789f656a794b48c7a1ae1261f6bd8bc1897d17d", size = 1502499, upload-time = "2026-09-13T18:48:17.405Z" }, + { url = "https://files.pythonhosted.org/packages/2b/e4/d974e55c2e247ef26ed1df01c74940583db9a5b3a8bcaad5732c6e2047fb/ast_serialize-0.11.2-cp315-abi3.abi3t-win32.whl", hash = "sha256:af8c003ce721b0099dd55cef4ba733500fc3054ea0cc8565d8957aaf7cccdeb4", size = 1119739, upload-time = "2026-09-13T18:48:19.005Z" }, + { url = "https://files.pythonhosted.org/packages/0d/00/d229443488e095054d5e0c0cc20689a2633b899d735849ff1b2c8e4f0cbf/ast_serialize-0.11.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:554d117cb916d8032d85007c654d179efbbfd446174c048062778136a922944f", size = 1158602, upload-time = "2026-09-13T18:48:20.524Z" }, + { url = "https://files.pythonhosted.org/packages/11/75/389fc1a6cfa0c4b2ce522f47d8401329d8bb11732e516d46465960fef1d9/ast_serialize-0.11.2-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:d60515335750d431e462af6e722bb55720a5e7827192777bddfd9c4376065a4d", size = 1128842, upload-time = "2026-09-13T18:48:22.052Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/c4f36898f19c728d091cdfdf960c9488e8d82bbe2e49ba13c05f43907a5d/ast_serialize-0.11.2-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:89499a439955931281986e97ca4dd3c064bf0d2e0027c0017344eb86667733a1", size = 897204, upload-time = "2026-09-13T18:48:23.695Z" }, + { url = "https://files.pythonhosted.org/packages/b1/54/f67120006fc73a55b6d057d4662d061fbb4eceafce3047c76ca8b382eb11/ast_serialize-0.11.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:daadf1c3e0224621607ffe16f1379e4bd372271ed2e1db8a67878f0bab3ef7e4", size = 1240734, upload-time = "2026-09-13T18:48:25.287Z" }, + { url = "https://files.pythonhosted.org/packages/9a/7e/8f2ab68bddbe58a66fbbaad87beeae3e7d7edddb17263d1fc423936cf34d/ast_serialize-0.11.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1844ed9a487fb3de7325c52ddb33f2918b66b65cd54d3f8d83d23785ffe99fa4", size = 1228053, upload-time = "2026-09-13T18:48:26.788Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1b/8a69ab68f4c1603819f0481d756abdd8caf27cec7f1d77caa71007ebe997/ast_serialize-0.11.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b17869f4ba261a5fa468a753328a548f4dbaf74b4eadae9e28aff66df7f1425b", size = 1292542, upload-time = "2026-09-13T18:48:28.295Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ce/872f2e00f0467c289e483f0a34543463347243a2d0632748d89fcee5e0dc/ast_serialize-0.11.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:feb16d9c2a720e0120c58dd5d6e7b3c7c86b43249b60a3bc212bcb8fa031e2dd", size = 1294791, upload-time = "2026-09-13T18:48:29.969Z" }, + { url = "https://files.pythonhosted.org/packages/3a/82/36277c12af861c64b375c316135d8feffe3f400568463a8d2b2de4c2c4fb/ast_serialize-0.11.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3109fe4805384effc8d0f8e41fbf875aa8f389af91b4348c1cfb60ea6e4cb82", size = 1567583, upload-time = "2026-09-13T18:48:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d7/ec643df91cea8bcbcb4e8011d6a8b08e5119b84f9554879f3e3c786d29d1/ast_serialize-0.11.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abdb3e49ba053c3486ac1263bee9f16cc9a4a8abd9f8c90bfc21e3669f3ad9d1", size = 1312878, upload-time = "2026-09-13T18:48:33.495Z" }, + { url = "https://files.pythonhosted.org/packages/04/6f/4c992cd7841ba589fefb14ddc9aff2f6db7f2a615d4074f9ad04115b5ce0/ast_serialize-0.11.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7004ba572f09be34342ccb98dcd4bad5707d3d81adc8cb4c3f685d2a2c51bbc", size = 1312642, upload-time = "2026-09-13T18:48:35.294Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e3/22aaa209c231a83cfea004fd67dee7a7a54da3f169c6c460b14b96887385/ast_serialize-0.11.2-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:59c25f47524efa052971b860e128b1add0c94ede7dd16b2962952c85c3582365", size = 1319776, upload-time = "2026-09-13T18:48:36.866Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f7/d4685fb54d10108ce44d3bc893ef670854d61645d47ed96d73524db90c23/ast_serialize-0.11.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3a367e0e05ed2d1b747ceb07aa728a8c204cc008b589127e9bd4f40053d7575", size = 1365324, upload-time = "2026-09-13T18:48:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/42/3a/250643ffad02bda520c50a9a5f02a5d43259a06f34ce393c91761d134d7e/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:00bbf1f6669f813b48925b759f7ae4591067d456d443924055cab386e7e0a719", size = 1467653, upload-time = "2026-09-13T18:48:40.348Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/af66a646b9b7f8fdec95ce83fc7b1fe538b06864bc79bd554ac4fae2e6ea/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:ec1c20f89c3e0d83576e3c06f79375ce936266591fe0d5fd969914af3185cbaa", size = 1571914, upload-time = "2026-09-13T18:48:41.968Z" }, + { url = "https://files.pythonhosted.org/packages/34/82/77a9714564b9e8800087a8afec41527c65c39e49282baae2ac847b9c1c6a/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:c58bb119b73657fdc5569692f316e1e25ca114bd62f7782eb527c6be438ba3a9", size = 1569862, upload-time = "2026-09-13T18:48:43.701Z" }, + { url = "https://files.pythonhosted.org/packages/65/06/fa77b52f46b9bd6dcd8ff2b880e3781f8c1a316bb1342bc3de92907c6f96/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:f739e0b601be7300c5697a2573d9200bd1db74b34ab111ef9537b9d5dcd7f106", size = 1699020, upload-time = "2026-09-13T18:48:45.261Z" }, + { url = "https://files.pythonhosted.org/packages/e1/09/239c83153c7e0798e5867d6909cb06f53dccfef02f6999c8e2e21ecb98c3/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:cae5addfbb54cc1d47fe947ef9138e9d83849ed1cbc72b819cf36d96a2315b07", size = 1492869, upload-time = "2026-09-13T18:48:46.922Z" }, + { url = "https://files.pythonhosted.org/packages/2f/eb/6108fb9a43fc7ab5529856e38e33c6e3e064fbfe375fdcbb208c7cd5438d/ast_serialize-0.11.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2fa3be25f7f5351b1b39c9f8a52779b2dbf21199efbae564b4746422e8edca4e", size = 1511621, upload-time = "2026-09-13T18:48:48.667Z" }, + { url = "https://files.pythonhosted.org/packages/8a/82/60367e58ef346a41ebc90d3f28593c1b8f5c2cb5314c7b2bbd98910ee131/ast_serialize-0.11.2-cp39-abi3-win32.whl", hash = "sha256:d70556a2f9230a44c99a655774cde823f056efc34466eabfb4085f0cb1ea9f99", size = 1125873, upload-time = "2026-09-13T18:48:50.661Z" }, + { url = "https://files.pythonhosted.org/packages/23/bf/b419c3205ce1143ba7c69baef4f0ba43c14d8712113bf34f9e0d27d609be/ast_serialize-0.11.2-cp39-abi3-win_amd64.whl", hash = "sha256:b9065dd23131a23b41f5bab3bf4e9b3c350a3fe8e36e8200eded9b729fcea484", size = 1165434, upload-time = "2026-09-13T18:48:52.169Z" }, + { url = "https://files.pythonhosted.org/packages/91/a7/c8bbb2173f7a7131b3b2412035b2d814ab5ef2ce9799bd06f07c451640e4/ast_serialize-0.11.2-cp39-abi3-win_arm64.whl", hash = "sha256:dab599cbdcb7b45b18c41fad746645580b3a24357082b7f0e8921cd373804f27", size = 1136031, upload-time = "2026-09-13T18:48:54.04Z" }, +] + [[package]] name = "async-timeout" version = "5.0.1" @@ -246,7 +312,8 @@ name = "caio" version = "0.12.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -628,6 +695,7 @@ apps = [ { name = "watchdog" }, ] dev = [ + { name = "mypy" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -682,6 +750,7 @@ requires-dist = [ { name = "langgraph-checkpoint-sqlite", marker = "extra == 'graph'", specifier = ">=3.0.3" }, { name = "leidenalg", marker = "extra == 'graph'", specifier = ">=0.10.0" }, { name = "llmlingua", marker = "python_full_version >= '3.11' and extra == 'models-local'", specifier = ">=0.2.2" }, + { name = "mypy", marker = "extra == 'dev'", specifier = "==2.3.1" }, { name = "networkx", marker = "extra == 'graph'", specifier = ">=3.0" }, { name = "numpy", marker = "python_full_version >= '3.11' and extra == 'models-local'", specifier = ">=1.24.0" }, { name = "onnxruntime", marker = "python_full_version >= '3.11' and extra == 'models-local'", specifier = ">=1.24.1" }, @@ -701,7 +770,7 @@ requires-dist = [ { name = "qdrant-client", marker = "extra == 'local'", specifier = ">=1.7.0" }, { name = "rank-bm25", marker = "extra == 'local'", specifier = ">=0.2.2" }, { name = "regex", specifier = ">=2024.11.6" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.8" }, { name = "sentence-transformers", extras = ["onnx"], marker = "python_full_version >= '3.11' and extra == 'models-local'", specifier = ">=3.2.0" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "tiktoken", marker = "extra == 'models-hosted'", specifier = ">=0.5.0" }, @@ -1547,6 +1616,136 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/b9/d84d6225d79fed4fee19faef49e8b8fa58b70fcf55de5061bb1adf602eaa/leidenalg-0.11.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:023e97e4c96573e89fdea876da663e38694f6fa91e2c85e98455101409381d88", size = 2570109, upload-time = "2025-10-31T17:14:47.585Z" }, ] +[[package]] +name = "librt" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9b/356320fbae2ac8467e21c5e73e1389c80468e4998c62cc7d3536cc51b614/librt-0.15.0.tar.gz", hash = "sha256:4e66cbe84437497d951b799d3e1551291b6fb3d643820a7014b3655d57a59162", size = 214338, upload-time = "2026-08-07T10:49:42.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/12/e2e9ca532cf5a0e08c9489826c4a35c6958c92ba0313fda70e8c6c3912be/librt-0.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e1a49adf16a7c9d9646816c2946135527197b6fcf4347c7b8b761cf1bfbf4489", size = 148673, upload-time = "2026-08-07T10:46:22.569Z" }, + { url = "https://files.pythonhosted.org/packages/6d/7c/02005e23478bd5950618d9712e0fd2b4c511657857f3efd8ba6a5feabcdd/librt-0.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81a398f45b45a59200e13cd5ad1ae1d3f44334de98b148331afe2cdfee701c52", size = 153547, upload-time = "2026-08-07T10:46:23.931Z" }, + { url = "https://files.pythonhosted.org/packages/a0/90/d8848a735f5642077fc4b3b4bebcdb08edf10178e3add45597f5201a368f/librt-0.15.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4eafbaff06b9563f8b1c850621ce51605de05208e09d4d71ce490bc972b7b9e8", size = 494355, upload-time = "2026-08-07T10:46:25.122Z" }, + { url = "https://files.pythonhosted.org/packages/e1/0b/8604f41ea02feace490e9e405a338a15f9905369f55b239a9ce31c946f24/librt-0.15.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:b0411b4066db926b80258c60dcb0e6db4c9cee312eab45b7e8866b17ddf9ada1", size = 485459, upload-time = "2026-08-07T10:46:26.447Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ac/84153bda1ce0da609182527ab92b40d961809e544eefdc5a1c2422971416/librt-0.15.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:febb1ce6cac545a54e6b769982824e955a700fdd9fbf3a08a3d82c990968b57d", size = 498398, upload-time = "2026-08-07T10:46:27.701Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3a/5ca6cd282b2c244bec8ec84102e09773264e9c02891d56ab3a8f0e4d7083/librt-0.15.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b230acc1c3bfe2d6f2627ba2b95dc92e58aa494600e9722d0e6ccbc931e59702", size = 515474, upload-time = "2026-08-07T10:46:28.9Z" }, + { url = "https://files.pythonhosted.org/packages/73/d3/bd34110234779eb843c6ed66aba7c9b2091d3dd85989f1fb9922f564cb7a/librt-0.15.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6da110e5f314c19ab8478464d02ae18808ae73d522c15260fa4918acdcd64da9", size = 509484, upload-time = "2026-08-07T10:46:30.124Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/43c3f7f071d71631a7daa3b835ef2168ea39f20692d81464d4e47fbaa6d6/librt-0.15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:eab9208b00ca55bf75983ec99f7bf13acc746a36102e98953addaad7f7ea1e1b", size = 532534, upload-time = "2026-08-07T10:46:31.511Z" }, + { url = "https://files.pythonhosted.org/packages/c5/1c/b854adf036ea817c40408873a5b794d65a91d9f0f39826f2ad2a2d5d7f48/librt-0.15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6c013cd3a1721e69e14380ada97eaa4b7b0cdf1c6b96fa765d4ea47c875088db", size = 537087, upload-time = "2026-08-07T10:46:32.734Z" }, + { url = "https://files.pythonhosted.org/packages/25/5c/c9a890e244e7dd725d3bd8b560e41f0aec787eaf343b46956a290ab7b841/librt-0.15.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:567b1c430f8bd560e689421468278ac5941bab4a05303b5d95b6ae10db03f451", size = 536575, upload-time = "2026-08-07T10:46:33.965Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c5/c8e70b60b704299555f55db468eb46b1c81bfc60201ffbfe20407d89870c/librt-0.15.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:29c4cab9df457b19672c39be7f384ebb2bc925c4e2684b8780c222b43eb36389", size = 517142, upload-time = "2026-08-07T10:46:35.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/d1/767a90c41f5d381b3195bc88ac0ec4afda35777c9c781e1f9848fedd965e/librt-0.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bccbd8e5b0bffb7106cf18eb1baa3d7194b1cebb3b4b1cdbd4bdb19382a6ee6c", size = 558714, upload-time = "2026-08-07T10:46:36.829Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b4/3c0624b8dc8301ab808f2b3a910995bcabe28df070fb9a0e5505ae997dae/librt-0.15.0-cp310-cp310-win32.whl", hash = "sha256:8ae493ed5f659a7761c43d42f183db514536073ded9bcf671d2d1df47e29a07e", size = 104426, upload-time = "2026-08-07T10:46:38.594Z" }, + { url = "https://files.pythonhosted.org/packages/31/98/e91c0382304bedb2db9c6801897319a9dcb68daac5e975819b562362f20d/librt-0.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:bc25fb356d0c7810bb49ff3df908ad1fda6995d660ab099ded69244ed7ab6053", size = 125057, upload-time = "2026-08-07T10:46:40.052Z" }, + { url = "https://files.pythonhosted.org/packages/59/52/06790ced2ac7117f890c21bda43c39c958ec82aa665c0718e821d33ff939/librt-0.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:823b92cf3c18ecd08afc70c42473888b41b6e8ef5046f3b82c05c154a2fa3d22", size = 148039, upload-time = "2026-08-07T10:46:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1d/8e150b7fc449a1f33c8a760965cc1f43b14fc1577d9d0b50ab2701420e74/librt-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70bc1b602cf59917e8f0c7a2cbc8bcc6fbc14d5486136b00707a79619121d63", size = 153067, upload-time = "2026-08-07T10:46:42.418Z" }, + { url = "https://files.pythonhosted.org/packages/51/87/a162bc5a66a35599dc619ecb215145f4de7d68e886b479b6d12593139f7c/librt-0.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:814ff83a25b5fce8b9c80c4dd803153fb5c5599fc74db9e022466938368957ef", size = 493087, upload-time = "2026-08-07T10:46:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/e5/3a/aeea1fc620cf48060d3065b37614edbf97043c099d0f50782bc8ca61d897/librt-0.15.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:57f5eeb6ad4c180de583b1038e61fe5fbd9796bb69a8a1c1a0c7ddbec4c8c60f", size = 485608, upload-time = "2026-08-07T10:46:45.038Z" }, + { url = "https://files.pythonhosted.org/packages/52/ff/fe571ad416f0856fd0d5578ffc2e6dc531891e586e36b647bcf50569cab8/librt-0.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82909c8f7eb9952656b65d3147afde4cf8e6d5a991eebc86418b5e65843b0ab8", size = 498723, upload-time = "2026-08-07T10:46:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/7a65eb5dedb1f00aebd948cdd8e17add48bf066cab3514e9daf84ab45a6c/librt-0.15.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f779070399f991400fc451719e0ea388eb7de313388bada2c127a35de05f798a", size = 516002, upload-time = "2026-08-07T10:46:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/59832b0ebfbd08c2742e6ece372ceb53f18bf1faef5d33c8daf3abebf749/librt-0.15.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bac89069bc496ebdf4f79ebb57bbd10d0b214c8454225deb672d91002bd17e18", size = 508607, upload-time = "2026-08-07T10:46:48.873Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/37fa73f3b43ebd8259f91ae9102a15e5a54e65d581e48dea72df3e81d7a4/librt-0.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e0d00c708fb2f5822b152429b1ac80a58dbbbc3f6c232c4d13a3f7fcf2ea5b4c", size = 530422, upload-time = "2026-08-07T10:46:50.45Z" }, + { url = "https://files.pythonhosted.org/packages/26/02/e046c6fe7a5881ac34623242192f484426ba8a75595fd18f22c53a3f530f/librt-0.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c6624fe268625869485553dd7cc1daf30d22558215bb2a4ff16f67a9801a31a", size = 534303, upload-time = "2026-08-07T10:46:51.693Z" }, + { url = "https://files.pythonhosted.org/packages/95/32/d5e6d861ab0366f3edf74f887ab0c9eb9f535aaf01d32b80b4f734daa179/librt-0.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f56b397858a23dacf35ede366ed2212fdc03a6a57a1ad36468ad6e9dc5fac091", size = 536084, upload-time = "2026-08-07T10:46:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/d69d725513fe53fc90c6d7a1f86e4428939bad2fb905b17fe4c18d413dde/librt-0.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4388184646efe2054911c5b00a1077d6d1ee86a95b7e8ba96dc7850a809f3f40", size = 514307, upload-time = "2026-08-07T10:46:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/36/93/f8aded0d6682b4f25820fa86e0690f87f01df9fd7bd09ddb04d9167ad021/librt-0.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:97335f59082f9fe2ce6c2a9cc6433a0114bbb6cd4d5c09dd76c95c68b9f9a8b0", size = 557686, upload-time = "2026-08-07T10:46:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/ffeb6bdeb6cd862b4272fddc8ad05f938dd25d020ed517e631813917d80a/librt-0.15.0-cp311-cp311-win32.whl", hash = "sha256:83380ffde38062a2e9bb55d83e74474f6614665528b98a6928720fc006dfffbb", size = 104917, upload-time = "2026-08-07T10:46:56.605Z" }, + { url = "https://files.pythonhosted.org/packages/96/28/7e2313a3ffbf0b4de7ba3da58a09e488507b4bd1ea2b5e69378354a23415/librt-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:f75720477ee05d509a310e856cacc8d909adc182f7b91193c207bcc26d7ee6db", size = 125886, upload-time = "2026-08-07T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/39/9e/04b8c3cde014ef255ee785730425268354543acc38902093a40afa0dc164/librt-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:256237037a3ab001ae8d9803b2d43562a4c3aa38739843694349e4d5ebb0fd56", size = 111885, upload-time = "2026-08-07T10:46:58.787Z" }, + { url = "https://files.pythonhosted.org/packages/ba/39/99c25030e782bdfb7a21be8c05254806a2e4bbb05c8d50c2a2130acbfa05/librt-0.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e87bc679f86a99aa3b26e3c78eeb821a247c9a28eae48eaafcc32c3bf4c3bb9e", size = 151021, upload-time = "2026-08-07T10:47:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/14/43/f4b1bd1b2888798a1409808889a25ea1ba49eaabce7d681ed27734c2df9d/librt-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71599e011ac880e8e45d46047d714871894c7d4ab6f25626f8d4f89da21f368d", size = 155267, upload-time = "2026-08-07T10:47:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/0c/db/3ad9c965c72f1e1d6beeec44ec10a54e17be8ae042fbb4baade16cbadced/librt-0.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c802434092b769b1d613ed2e13fac15fbfce1934a74bd10283b03c0fae231cd1", size = 503136, upload-time = "2026-08-07T10:47:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/5888a6d76acd62ebce66c61b74d94e9370b9c32929f111e487bb6546f8ed/librt-0.15.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5500eeae393a184d14e1f35645962c27129d20c81afa4069e6ef826ebc2b3aaa", size = 496670, upload-time = "2026-08-07T10:47:03.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/ab57cc2f5b276156da02bb7f5a8921bada1cb1993ffec99acf811c602c23/librt-0.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6ecfc32dfb46fb7b565bcd6abf9412acf978775a998273d22888a6d7953730dd", size = 513688, upload-time = "2026-08-07T10:47:04.981Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/bdbb0b648b5c2befb031f4c6f3b1dd857415e8fb492a25a3c764a6681e6c/librt-0.15.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cc46cfd15022e35084355478c9ac809d90b1152222706ac9a7655ec21df6fa", size = 531904, upload-time = "2026-08-07T10:47:06.211Z" }, + { url = "https://files.pythonhosted.org/packages/93/26/473c2e4b6c104e9e58e27ce95fc8005c8bd4fc36cae4f254371125a92db8/librt-0.15.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5f51401d102c885b9ca509e62c79b1dbff286e1b9b047fde6f763780789356d", size = 524427, upload-time = "2026-08-07T10:47:07.592Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/03b3abb82b41714671b907bf6989b228e31e6a8af52dec82b5b0728dc250/librt-0.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cc30523e3f1a23fb7511cc659834a0d01a1042bb9de359bc1c131cc4ec6c9656", size = 543155, upload-time = "2026-08-07T10:47:08.866Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0e/9bb1f0a4affbd0a1888f4f79dc03ed2a299d9a2c26c59ab2a97dcbf11903/librt-0.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:59fe030d8ae4a57e3fb7756bf35a858de74e04066fc8555c53d0af979132af81", size = 546890, upload-time = "2026-08-07T10:47:10.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/84/6937a280d461f7de6e031ffb02edc2b7c3c90d49d630565ce8ff27cbc5f2/librt-0.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5a6526a2a956bbb1e4ae3568c82e650fc99119c66bb011ea60715744955a2b4d", size = 555163, upload-time = "2026-08-07T10:47:11.798Z" }, + { url = "https://files.pythonhosted.org/packages/bc/95/2a2853c1ee014bf102116e7f897a04beeaeb2461b45b79af98bdfb95f1ef/librt-0.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:85ea21ec6730194d67156b0e0b5430ccb1d61f8b8b907e39b37f9812b74a13f0", size = 535812, upload-time = "2026-08-07T10:47:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/cf9601c1b4c5f09280acd5d83abdb2e68527a2be8257136eb42304218622/librt-0.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1e47b8ba865d7ede071a91a7163073bbaeb72541f1ef8a07d512c45c7b5007f2", size = 573688, upload-time = "2026-08-07T10:47:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/47/6d/9ac7cbec46189a7625af4b5acbd25f10d827f4141b2002181848c8418923/librt-0.15.0-cp312-cp312-win32.whl", hash = "sha256:a5207ec414d1c4a2a7231b2086970dc036f94293cdf338190984958a013a42f1", size = 106138, upload-time = "2026-08-07T10:47:15.973Z" }, + { url = "https://files.pythonhosted.org/packages/38/d0/2ae99c83be86ce23f925ac1aeeedc777e97f427c4a8d190c70d0a16e9a87/librt-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:73b30cfa976659b3917c8f6153bdb0591c6a9ec6583599fd24a689b690622022", size = 126974, upload-time = "2026-08-07T10:47:17.049Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ef/dd24f9635c730b86b87587967dda7516b1845e8b17684603d31607fed598/librt-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:a54cf9e0ef47b96af580849db5471142200568ce1e02cbf416addab551369570", size = 112292, upload-time = "2026-08-07T10:47:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/467b53a601b406ccd7b97c1fd54b59cb34f9185ad5ce7e9d5c3c4e8961c8/librt-0.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db13ca398005abcbe538deda87b686d9bd08b7001cf40c4c06b444960ae10a26", size = 151029, upload-time = "2026-08-07T10:47:19.312Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/36c2299b7a94b84fdd01220d8a777a71be5be0925bb0dbdf71c0a06a34d9/librt-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa1f1995789dca3698bc550aaceb09a51bd5df0a057ff84ff15296cd1975b801", size = 155194, upload-time = "2026-08-07T10:47:20.398Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/ed5071f9325845e670bd36012757419767fbf56af77ed483077b9e4db541/librt-0.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55456ea87d8df21808446d03817be2f65e20391c1c615d9187440dff28cd08dc", size = 502568, upload-time = "2026-08-07T10:47:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/6450c67c3615d87704bcbc21323fafc69c799b06a044c447529f725d4b01/librt-0.15.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5a86a5a08c2235316bdb359d5dbb6ce0abfca7fac06363103e2c5af571d92f95", size = 496153, upload-time = "2026-08-07T10:47:22.925Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d6/5f52b722bc75076954b3bfd49be15ea362df4d580c6fb315d0f617100d30/librt-0.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56b6a368529bed262da40ce13f8fef590db0479819cca84f16a1f01ac356d0b", size = 513336, upload-time = "2026-08-07T10:47:24.213Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e2/c08fd1d36ce63ea5a12b85c5d37f4550b5f86a692167e41e5a74222607ae/librt-0.15.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:234d8d394721fa0d786af15ebf1f3fb7f3ed82fd1cd0cde45c2f247b5d4281d2", size = 531661, upload-time = "2026-08-07T10:47:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/d9482fcbeb177b9eb87bb3899eeb3b42be690313c652f9e146b1d0681fb2/librt-0.15.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d8363d7accb0286ac3a0e633f396e93800dafb8150494505daf9515bbda591f3", size = 524487, upload-time = "2026-08-07T10:47:26.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/075171517b41f861753034fbb151b42cfc83bcc853849f24f5e66fd60ccf/librt-0.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0f0ee3644d951f31055ad07d77d92520e84505dd7a432cc4cd501dd70ee06785", size = 543201, upload-time = "2026-08-07T10:47:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/b0/03/42c2330f37eeb475b6affeedd06518f60035f323af3a839335e3fc9fef2d/librt-0.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2cfd1a81a648806e6a7717be4cc4d1bb392fa229752bf8444ba365e381e984d6", size = 546467, upload-time = "2026-08-07T10:47:29.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/1e/1ad4c5638f7e64d8560328bd25c54b409a661bdb6ff254b38ff90744288d/librt-0.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a6cd22c9da0d866558e46a041f1cc0c2bbb26b61b137b2347fa834c332e1d101", size = 555139, upload-time = "2026-08-07T10:47:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/39fa7d15db1204cd1cbe6514680fbdc243adf754a0885061308f43afc013/librt-0.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6d5225ef8801e4ea5e482fa9b5dfb891dd9ef6f6d870f1f25d449ca2c70ac218", size = 536050, upload-time = "2026-08-07T10:47:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/c6dcf0dd8e26dc0c9a499a2abab8646c86dcaf9ecea9524cb46d3686331a/librt-0.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d28a05796b99f749bf8794f17ba9ba1612d0076b802e9cfc62c554634e9ce3b", size = 573700, upload-time = "2026-08-07T10:47:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ab54c71a7918a7c34fa5327fb61390a77446a07a146fbfb1165250a61035/librt-0.15.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2067ff438048cead9d223ca5675bae2a25e520a7c3e6c1498bf9c6892d22caab", size = 82194, upload-time = "2026-08-07T10:47:34.835Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b2/4f9a243bb892395f3becb80789ade13771701091f9f07ab8230247953ba8/librt-0.15.0-cp313-cp313-win32.whl", hash = "sha256:1cd3b721f24c206398b9e26da3c3a9c011e6e89d06f318ba8ebefc30f1003890", size = 106231, upload-time = "2026-08-07T10:47:36.251Z" }, + { url = "https://files.pythonhosted.org/packages/bf/af/64aff4885a40b93132382f2c314647d722574605416504379184ef3045ea/librt-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:f395a4a9a03ac062dbe9a9f82e0c720502e590a38feee6a757bc82e9c63afbd8", size = 126996, upload-time = "2026-08-07T10:47:37.453Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/335bccf6c7cb9028cb0b54aead27d9ece3f01f83bc6baa2abace5da655c1/librt-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:0a15cb554761247d84a3ec0cbdf4078d70725384f0e4662c0fa3b26266eb60ad", size = 112188, upload-time = "2026-08-07T10:47:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/a8/93/949053fb462eecc4a9a5ee770a81f4b40be7b79538b245545d4aebc6b58b/librt-0.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f5de7feedc56337a088eb15cd9fafa9938367362221d8cc62c642b7f94821993", size = 149833, upload-time = "2026-08-07T10:47:39.86Z" }, + { url = "https://files.pythonhosted.org/packages/61/ca/8281aa6cd560a3420e4497729f6b704b53be3eeaaef82d5aeadddaf7441f/librt-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c0eb900c0e91f4aebe680845242e614f1864edfd44106380d0752ac29522bf8", size = 154088, upload-time = "2026-08-07T10:47:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/02/1a1662dceaba6a086360891448d5ce9a7d3555976cae59a31a39d744b9c7/librt-0.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8c9a650a188e38bac005048cbe6342e81407782944d01934540ab75e417df21", size = 494215, upload-time = "2026-08-07T10:47:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/99211619dc656370a3740c33d2b0b6d5a3fb1e73689314f6ed477a397dc4/librt-0.15.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:92bfed8deec93df30286b9fe9e3b1dd17329cc076a192b4ee5ec223841d54953", size = 491173, upload-time = "2026-08-07T10:47:43.683Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/5448d0b05f4579b635d3899176817ebf561af0e57bacd425b5b1887264c1/librt-0.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec4b19788f835711a2072f9dbe6b03b3bf32ed1f0fb30cf399bdd59d9f0c33fa", size = 505512, upload-time = "2026-08-07T10:47:45.314Z" }, + { url = "https://files.pythonhosted.org/packages/95/82/01940e40b83c43a546c4a3c896cf34ca272a9690899d55914e4827b3dcce/librt-0.15.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4c7bacb70930f3d0a56f4ecf1be474a1f0d941b01dd73b756f3c256d42cb879", size = 523073, upload-time = "2026-08-07T10:47:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/88/fa/759c0030f3ee371439eb26de34fc745807caf0abb878af7af4b8b7c3dd3d/librt-0.15.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e79f05e4a08b4d880342673312bbc895b56df7765605796f15902eb5367d3ae", size = 515080, upload-time = "2026-08-07T10:47:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/894e072228fcb159703c655da69f8cd10dbed489c36e3df7dd032a2483be/librt-0.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a417149c0cba4d50b61e992e5a15e69eaf96746609b461cc4ed168aeef6b79dd", size = 534164, upload-time = "2026-08-07T10:47:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/98/a3/0078e91c1f36f8815db17827de15650b9a3fe56c55fbf998c854b34e40d3/librt-0.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:da7a94d6a3411f579d72aa3e3bc5fbca7ed4549f3dbd7e5de3aa567333374285", size = 540616, upload-time = "2026-08-07T10:47:51.408Z" }, + { url = "https://files.pythonhosted.org/packages/86/33/81a29b796dd52a45e9ef7974c7732926e8f10f15b8d2be505665979f896d/librt-0.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:856f743ae607f2c1380eccb566c0038a9fb3eabf0fc2be2704d76d9f73557239", size = 545890, upload-time = "2026-08-07T10:47:52.818Z" }, + { url = "https://files.pythonhosted.org/packages/05/82/8be1baa1350e5d30cfd70ae79d0a6f4dc5862ef47f7bb2808aabc9bb86e5/librt-0.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:779a6e7c894737e5983e7790a9c78c4000c30e23c9aada08081bdbea53b0fa60", size = 523287, upload-time = "2026-08-07T10:47:54.165Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4f/d1be6a01a35c20ef734e0e44113f87d4af756a9354a89dcfbe3b4f8af5e1/librt-0.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96bb17dbe8bab3c0954fbebfc69ed395599de75b6bbc35e3270a878e15d4dd65", size = 565868, upload-time = "2026-08-07T10:47:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/67/88/649cfa33f5825927b160610f670bdab012a64d627eddb94fa795ea4292fd/librt-0.15.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7220697efaa6e5348fc3d18ee7f8563d4bfecd9872b37ffb915bfc1d08840622", size = 81619, upload-time = "2026-08-07T10:47:56.886Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/8e88a8d5e48fc8d1a817787fb6811dfff6499acd6c8683dd83934aa6ede0/librt-0.15.0-cp314-cp314-win32.whl", hash = "sha256:f54598964d357b1c5ab77cf5d92f21e598fe0e23cdbe9618480807f81b4eba15", size = 100138, upload-time = "2026-08-07T10:47:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/80/92/20fd6c4b6a1b1a564b076d55cd3d427d8428217d7638dc25a654cc4791d4/librt-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ff5893a2c23d886aa9ce786de5ac6ddc74aeeaf90743682b74d920e117d2e28", size = 121258, upload-time = "2026-08-07T10:47:59.564Z" }, + { url = "https://files.pythonhosted.org/packages/fc/28/6af430b44d9ebb897b865a3c363b6dcace51357be2347cc0f8f869656a86/librt-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:3722a099730704c9a3d70c879fc0f51daec25fe5f1555672d97bc595abeafb95", size = 106467, upload-time = "2026-08-07T10:48:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/7e/aa/b42bb798942ced219f6d63b27e07f91237887a8d0bd0921666db79a13790/librt-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:38c0c7d4b6fc06c3324b3f9162c8391bfc4fd9dde53afe1033ce7edb48d5a714", size = 159523, upload-time = "2026-08-07T10:48:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/75/03/1b53cd4ef904e73b1d828a5f90143bf94a2967d7cfff0b9ccf93e12aa9b4/librt-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b2fdd7ead3c995c37940a790690660d0ca006c302db26cc51933f6766866fc3", size = 161638, upload-time = "2026-08-07T10:48:03.725Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/9f9c9fba097d49e9e694c2b4dc331df31884645ecbc58a93b4b5fc69d2c5/librt-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fde98cf1fc4bac144ce23c2c4c017b924ba714509ea9334977b0b27050c837d", size = 701795, upload-time = "2026-08-07T10:48:05.135Z" }, + { url = "https://files.pythonhosted.org/packages/4c/05/0966840bda0380c8ae167b9043c6230202941cc90ea29c48e096964c765e/librt-0.15.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:e3b461183c5fa7681b48560f91515f53a953122fb30c71e07abc67d7ddf58c38", size = 682147, upload-time = "2026-08-07T10:48:06.555Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1c47ca573c30ea47d195aec26133af522fea1104afaace028d7b32247ea8/librt-0.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4bbcc257e3babea20a91715c361b24554ec4e8f51aa578568afc230799fe1a19", size = 696397, upload-time = "2026-08-07T10:48:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0f/1aed6223d4f9f9d1171a8596ff100ea4c3f7699fea7a4ba657c3e60daa6c/librt-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b845b8d48088fad0cadc84be4b8fda63203be7e9237b71015b3925443c1f35ab", size = 722542, upload-time = "2026-08-07T10:48:09.569Z" }, + { url = "https://files.pythonhosted.org/packages/c6/22/9e3a929aea456c97d69e6ef3884efea56d4807f97399471cc946baebd8af/librt-0.15.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b30e600e8f337b9bd7f39b86d9fdfedc73cc46e3d0f745931a23a234220bb7e2", size = 729709, upload-time = "2026-08-07T10:48:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1b/c327ef6018e3a9ca0b8e7c5eddeeb331ba8f9b76c24e126d37d0f6d62faf/librt-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:64b0c8c35aa4c4ed79896359f3e0b285cbe4e610042106500da4811c322cc108", size = 752891, upload-time = "2026-08-07T10:48:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d1/d5f1ea02c56930087009e39db9b70660a663e76c730b27b925d786718457/librt-0.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0da0d94cb802f32a0524653e7201f2cef72d5f700a5407678f5290483d4fcd08", size = 745301, upload-time = "2026-08-07T10:48:14.55Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3c/5f7c585d15ebb2250c73e7c0ee4e9e47be72c65d520c07ddbcdc62037674/librt-0.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4a6369168d371207339b1e50d4532b06a7121586141f82599505a3f315751d47", size = 747921, upload-time = "2026-08-07T10:48:16.453Z" }, + { url = "https://files.pythonhosted.org/packages/7f/52/1443a446486eba966bcbca1696b472e4f210320ec42f490a47f48fbf0fdc/librt-0.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c434e072557ade9cbc642d052c89d031efe47d5c9614523619d0d74a02378e81", size = 727561, upload-time = "2026-08-07T10:48:18.089Z" }, + { url = "https://files.pythonhosted.org/packages/79/91/2270a9380f11725cf83ce1925a5e32dd1dde2be9bba597f25c10a38644e7/librt-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7eec6a42018bc1d45763b1c162d3d2bf7c3b9a1b0ed30d3e91dcba390efefcc", size = 774417, upload-time = "2026-08-07T10:48:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/f4b1548d4f5b99186737fe27aec238e9823e8d5d23bf4df007c030689dc5/librt-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:6912fa5e635d74529ac7cdb1bdf6ca3af4453da8d1edbe0110ee1cb4ad407ebf", size = 104381, upload-time = "2026-08-07T10:48:21.048Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/134afad262def1de04c0843c376d02135f1168af43f22e09a52bd8394727/librt-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8e11699ed745931c395acd3621b07062e0f840efa6935aad87a64ed0995f0915", size = 127034, upload-time = "2026-08-07T10:48:22.561Z" }, + { url = "https://files.pythonhosted.org/packages/99/5f/1b6846b20572bd699c9e9ec321a5f781845bee477df2aa2a43b28bc40119/librt-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5d2a91724463bfed4f573cd7a9fdc856d2e230d0c0e5a61416a93481dccd8605", size = 110827, upload-time = "2026-08-07T10:48:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/c6/44/4de9f4ddadb009a55c7758eb5736d62534a7daaf27bd71bc50e64b606b06/librt-0.15.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8443e38dcfcfdbcf5add5118c623efd788d65ac2e25756d6251a54a06a4d0aca", size = 149843, upload-time = "2026-08-07T10:48:25.148Z" }, + { url = "https://files.pythonhosted.org/packages/1f/eb/5d9ab71e30119c44094e0275f38b47dd327aea0f843a080396677029d508/librt-0.15.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6d15a29033c57490cfe2069097c6fc4049e4e65ffbb749be7dc453b7c4c68965", size = 154510, upload-time = "2026-08-07T10:48:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/8505d1b8f5e8c19587bd03f7429993b3e9ce5c06819d856bfb11d919374c/librt-0.15.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2c05c729b589e734c09578bf5964be48a911765484840d017bbc84f49d4c4ad", size = 497543, upload-time = "2026-08-07T10:48:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/3a8390775cb095765aded027ac9c63e7c8ea74e731498607544c6505de0e/librt-0.15.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:fa60887537e1d0cd2d9982269d33a709bf54b195cd2b9364fc0a758022af5bd9", size = 480452, upload-time = "2026-08-07T10:48:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/e7/40/258a4a7117ee915d66de5cd9b8ade65a440993161107ce3a686f1859955c/librt-0.15.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d8bc24219b24c0af375718942ab75e3544b2763085f40f965be4326734ae8328", size = 507768, upload-time = "2026-08-07T10:48:31.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c6/2f4dd296c97a0b85b98894519b279408ec9dd602d4f692b1ea0e25dee670/librt-0.15.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:86a21a7bd3fe3a419512ef424cc1c020f6771d0b29cfddff36d1635a855e63f0", size = 525122, upload-time = "2026-08-07T10:48:32.7Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/29eab42be13b2bf0ea8cb227135a45d44693e30a7e8b92871981ff56b82b/librt-0.15.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dbab647e88d90b3167b91efe7091e248653688ed4337e4f90907a722c7361bb9", size = 520371, upload-time = "2026-08-07T10:48:34.294Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/4bad71adeca8fe208b775c2a35417fa5a2584c8f4791daaf89a89450fea1/librt-0.15.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d8edcf6f550e918dca779c069b9e156385c60b406f99fc7641f32c52f7193659", size = 537258, upload-time = "2026-08-07T10:48:35.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/63/59dba6143fdcc7240c54458b629f3250000a61b8945890fc9efd451b19c5/librt-0.15.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:8b62076030baa2d8b1501a46bf0e19c27a489aa90671c55665bff7887f7660b0", size = 527432, upload-time = "2026-08-07T10:48:37.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/21/21a24c6a2327d8362580efebe77286bf47b0f4062ec5ea41766e609d3c7d/librt-0.15.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d00d20d1818e82a07a0ee0aa89a98b17ed7916b92441090b683719cb20a59b6d", size = 548108, upload-time = "2026-08-07T10:48:39.384Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6d/fc68c89a7971418b41f9a873623ff935cb864097544c6a2f8ce491c8ef5d/librt-0.15.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4e6ee93fc3cf848dcbf0cce2eca73d8e7dcd0cc2b6df3a529d57750b30a4c55c", size = 529681, upload-time = "2026-08-07T10:48:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c2d98766124400d722063a630b0fde38a9fc768705d37eecca15c47dc192/librt-0.15.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:32896a0af72508ea979e0acb4e4c04cbeeae04938167950d535c83c45597167d", size = 567736, upload-time = "2026-08-07T10:48:43.124Z" }, + { url = "https://files.pythonhosted.org/packages/55/6c/f8c34a95e3a515c6e1c192b89511e7253c89a7760c6b500d57ffdb8d2dc8/librt-0.15.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:ec3ba415afaf951f6951b1dd16d3c8e4f540065fc382d7e70b823a79567ca374", size = 81673, upload-time = "2026-08-07T10:48:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/e23fa8e78679ec45728188650b39e8ff476c83b691c96f749217df3b1b7c/librt-0.15.0-cp315-cp315-win32.whl", hash = "sha256:d2813ba2503764f0450680c533d13df7cff9b49df1411062eded5f67db4195b9", size = 100081, upload-time = "2026-08-07T10:48:46.171Z" }, + { url = "https://files.pythonhosted.org/packages/e1/dc/3eb4c5e297343f0620a55532cd7c8d764d3001fa2159212dadf480464827/librt-0.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:b87d67e33afaf265262f2a66db578284b88ee2e6fcd224579cb5c15518677ad8", size = 121228, upload-time = "2026-08-07T10:48:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/97/70/43abce19f04e49762f8ec834c8fafee13cc40fd6b94a72a24e534febfcd0/librt-0.15.0-cp315-cp315-win_arm64.whl", hash = "sha256:713bd7df21170b982e729e46870f31d6b437bd1a9b4648cffb529bd3c2ec5c4b", size = 106487, upload-time = "2026-08-07T10:48:49.095Z" }, + { url = "https://files.pythonhosted.org/packages/de/15/83f2deddb9368b8951ec8c9477269b5b9b8bd9bbf15e57402d0f38817dca/librt-0.15.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:3de789c82752730f94782a5ee518baf9c05edf85733aeaf73bb6e518755cdf54", size = 159448, upload-time = "2026-08-07T10:48:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/043097353f9b3c73b583d07f6b8e552795463f4bfc8caf85e42eee50c26a/librt-0.15.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e0b5deec9a8664eb722c797241970fd4aa1894d25fda36a1ddac0f7407606bd6", size = 161686, upload-time = "2026-08-07T10:48:52.174Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2a/8ae77f9719d42ce71cd708560a3557b38ac3c17a0383e57f87084de45bbe/librt-0.15.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5563302a8359bc2295bb7084d1a8ed1519df96afb30eb2aa4e0bff7b54228988", size = 710668, upload-time = "2026-08-07T10:48:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/c0436ea134deb9a0d6da80a396a2739a81cb31e0418f7227239e23140898/librt-0.15.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:22d6263b9d39d7bbb286fa791945646e3218f1be2d693e36fb630f1d0e59cd13", size = 679396, upload-time = "2026-08-07T10:48:55.645Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/001e0d99aa9250d5cd5715a9081291a20656083459f9019cda15255329e1/librt-0.15.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39ffd14646190c454f0d86e0d256b33f00a87a26ab410e619773b841d0e41416", size = 704313, upload-time = "2026-08-07T10:48:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/2d/53/b34fa9d0ff00f136f4d58ebb4c411ff634baed1eb412bb602a2bc8dcafcb/librt-0.15.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c47318cd3a61401452de11282242937e3e057c4fd3dbaf601e269d0928a06c0a", size = 729847, upload-time = "2026-08-07T10:48:59.231Z" }, + { url = "https://files.pythonhosted.org/packages/86/ac/fa4d7a424665040e95baf480a6d523446057684b6758624c85338e8a23b2/librt-0.15.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a56a1d4f859a82ca5b99fc4b82c9b027b15e3c455c5cd99e7d0719f27bb20b6c", size = 742736, upload-time = "2026-08-07T10:49:01.151Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/e17a9bb5de6fb8c3186ed1a7d68d21618b027ac2d3633e03d3b6109c67ae/librt-0.15.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:077471b3182db4e17c36ae91555f36a4d2c00080b267f749bcad34a478a9a302", size = 763454, upload-time = "2026-08-07T10:49:03.039Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ec/ecd02cd30935b931b9cdbfed6ab5a099c51b280b4e7baa274da80978ed27/librt-0.15.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:411ca4d1b905b860ceba7570dd6717a71dedaddcc4b0f77ece710aa41ee11f8d", size = 743296, upload-time = "2026-08-07T10:49:04.941Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b5/b3c2b8353ce820a4854f78d19321344242f89fa71c975b71132ba9bf242a/librt-0.15.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1256589e0b0adb31751d685a68bce29d73407ddf4ef05d4188f49d5dcf9566d9", size = 756217, upload-time = "2026-08-07T10:49:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/3c/52/6cc22542ba59146b05cca2a656f9ff8bb67e38e63d12c3b0cc183d837bf1/librt-0.15.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:f42b74a53e5f26a0ba0007411a7455b66c67ce4022a39cc1f56fc4efd65bcbab", size = 741934, upload-time = "2026-08-07T10:49:08.839Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/a04b72b1aa86e3be23b2ecff8c1aad2dcc955bd3956d6d26e7e34267e57a/librt-0.15.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:291bf73caf78b9e88d6fae9bfd693207ff7d832e2fdbe2cf8e746bc13f5f892b", size = 783763, upload-time = "2026-08-07T10:49:10.661Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f0/89eb11dffbe9279ff37144dec786927314502ae0b114f1449dc78c458aab/librt-0.15.0-cp315-cp315t-win32.whl", hash = "sha256:c16d15ee371643ab48dc8248a3e680ebbeca573a13af2c3dd0c985b142d77162", size = 104313, upload-time = "2026-08-07T10:49:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4a/1f1978c200f563beda63c36adff2d65bbecb81e365e8e69e572f5f70fbc6/librt-0.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:dbd605739f228912dc49027cb764456b9757750bdc2b6b7773164db7096c6fd1", size = 126889, upload-time = "2026-08-07T10:49:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/800800bfed7b1fb10fc3f3d557785c3854e80d3f7a9800d784b176a1fc2d/librt-0.15.0-cp315-cp315t-win_arm64.whl", hash = "sha256:84d244b00604d17df3fc7736c327892d6bba66181254aa4087be807b6c342bdc", size = 110700, upload-time = "2026-08-07T10:49:15.499Z" }, +] + [[package]] name = "llmlingua" version = "0.2.2" @@ -1763,6 +1962,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] +[[package]] +name = "mypy" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/6a/878cc1097d4035f82bd516658d0c528d2a9955bc7b363afcbd0b07fea11b/mypy-2.3.1.tar.gz", hash = "sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419", size = 3992554, upload-time = "2026-08-15T03:03:38.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/b9/de8f67e12d721cdcc8ba6cfc440b989a4ba4dfabe4402ae94dfdd8bb30a4/mypy-2.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:57a936373fc690c43a8cd7e7e12a35148e4ec5aa7698ad7fc0a9f918bdc5be41", size = 14015541, upload-time = "2026-08-15T03:01:53.104Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8a/9e746ab012c67ed8ea3232a613716c306ee8c0b5682c80d8103b4f04568e/mypy-2.3.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d00d769056bde2f4e69c175071eba45cfb44fa1ed92bdfbfe64a93e0543b0cf0", size = 14248142, upload-time = "2026-08-15T03:02:43.201Z" }, + { url = "https://files.pythonhosted.org/packages/f7/5c/c99ff2d8d0e2c53393e32dfe22d9aa43a5d959d30db46c786dafd24527d3/mypy-2.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2166b29228835e1f88ff411e96639e6ca3c7fdde84b62ec211f70f86b4051167", size = 15193309, upload-time = "2026-08-15T03:01:28.714Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/124638f745243faae1ff4b37d5426fe41c0f0454535edc82fe8102b56a3c/mypy-2.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:83d36c2924df7426333abe7faf4724a7e1aab0d9fd41625e81b4683034b80c13", size = 15498246, upload-time = "2026-08-15T03:02:46.29Z" }, + { url = "https://files.pythonhosted.org/packages/b2/83/31c0781e243836505c0fb5f4e865487d6df1023e4ad959f4ebd4b84a0226/mypy-2.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:f12fdb70459d0060dea40b29e52163a961b156106d68d57882a6a9f648983a53", size = 11155028, upload-time = "2026-08-15T03:01:39.08Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ab/bc2eb0129e72d7d7d93d5e981a78084a9abefda7efa732a7e02f97d6e27d/mypy-2.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:e099200a1b1b1223a4951f0a90cbff1b8c91b250ba599dab1f7217a628144d90", size = 10151438, upload-time = "2026-08-15T03:02:19.04Z" }, + { url = "https://files.pythonhosted.org/packages/a4/be/c624d4241484f37dc62839e177ab607a9b8b3e96f0866544ca99e8e41d51/mypy-2.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94f04929f1c44c35fb0061e912087edaf504acede963a4a7d00680bd089d8531", size = 13936739, upload-time = "2026-08-15T03:03:26.475Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/e3cf72f90dce5960871c82551c8fba6da05fc1018f79be41c047bd126bdd/mypy-2.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5d716048611e85ca9eefb2e1baa5d73ede389b5820ded260ea27c757d667af8", size = 14166460, upload-time = "2026-08-15T03:01:50.565Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ff/6b97d58aa0f79a5ab9b472db1f6d6df1b11a51d74d0c08ab3760d3a613ba/mypy-2.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b091a455111214cb5c9d54a57b9618e9a49f9fe2a42e4e1ac86e9d104ed96ce8", size = 15100476, upload-time = "2026-08-15T03:03:12.079Z" }, + { url = "https://files.pythonhosted.org/packages/da/f0/cbb4b7d2ae3ac635f6b4f2d9b04070b8a92edf50da599d3b39e5ed109001/mypy-2.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df12e20c9efd614738c71b390007ecd0181125afc4ccafca04d78a1d2eed2c01", size = 15347826, upload-time = "2026-08-15T03:03:02.856Z" }, + { url = "https://files.pythonhosted.org/packages/5f/10/91dcdc6f8d43fc08e6a06ab1f9732f3abaaf835ac1b2e67b9dff56910855/mypy-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:52eaf3a155f35cf80b40220288c861eb45f14a2340c1f6cbfbdb0feff32879d1", size = 11142615, upload-time = "2026-08-15T03:03:36.316Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8a/28d54535bf4b9aa43b2d8918c2ef660378b9f66b23d78dcee052744ae622/mypy-2.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9b4eacbee8a69836c06eff6d0dd4e134a07c2b047755b30c08625fe214f322c6", size = 10141145, upload-time = "2026-08-15T03:03:07.406Z" }, + { url = "https://files.pythonhosted.org/packages/85/da/d6effc4f808a842d91edc22535dc9e799d2ff6e91449168b7f47a0771f54/mypy-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff", size = 14047547, upload-time = "2026-08-15T03:02:57.707Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e6/478229701dab76f26485fc8ff5d6f241f393da22447400bbc56f6946aebe/mypy-2.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff", size = 14216515, upload-time = "2026-08-15T03:01:26.496Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fe/7c42327a3b21e84681f691982cbfe43f334a3685f3b683b72c376476c4fa/mypy-2.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080", size = 15307789, upload-time = "2026-08-15T03:03:31.62Z" }, + { url = "https://files.pythonhosted.org/packages/59/f4/7e597edbe01b5a56fa958ce541302dcaabfed979966f1dffedbea0ea0fc2/mypy-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355", size = 15548831, upload-time = "2026-08-15T03:03:15.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/52/cb31e084bc0314a1e384bdd677a4b80e55af04ccac077545e2238b9d320a/mypy-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb", size = 11226359, upload-time = "2026-08-15T03:03:29.002Z" }, + { url = "https://files.pythonhosted.org/packages/7a/47/88fcf6217b43fa2da81a8c2611370af18141536a4f0294bbf98b457d456d/mypy-2.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b", size = 10214707, upload-time = "2026-08-15T03:02:48.807Z" }, + { url = "https://files.pythonhosted.org/packages/de/cf/862010ee800ca9c2bd0c4c0dacf0f092e5411824a09b8f97ad4be8fe250e/mypy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d", size = 13964542, upload-time = "2026-08-15T03:02:21.43Z" }, + { url = "https://files.pythonhosted.org/packages/75/5a/3f3a2107b41e3e92e617e25daaee121413b91e9784bea733131ed4fecc5d/mypy-2.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb", size = 14168922, upload-time = "2026-08-15T03:03:00.351Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/04dc4fe7e63d7820fa4eff272e95157d30cbea921388f3ab3fe77794cd0b/mypy-2.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226", size = 15244791, upload-time = "2026-08-15T03:02:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/96/fc/c3053b26b9054949285aa868cb6af8c10e7591541cacd79c5dcc06a1fcf9/mypy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74", size = 15501627, upload-time = "2026-08-15T03:03:34.128Z" }, + { url = "https://files.pythonhosted.org/packages/70/4e/d77daab008bbc4e5001374d7928f4a260d28f0e6747af444fc4763f7a310/mypy-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6", size = 11243961, upload-time = "2026-08-15T03:02:11.952Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f8/7eb68c136e4abd30569fe31ef2bfcb7eceae9952cab80017c04cd09f5d0c/mypy-2.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac", size = 10213219, upload-time = "2026-08-15T03:02:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/42a49d44aeff804edf1b19acce0b49e8bd1a9c57dee9605dd8d980aa43d7/mypy-2.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d", size = 13986778, upload-time = "2026-08-15T03:01:33.69Z" }, + { url = "https://files.pythonhosted.org/packages/45/13/9331fd2dfed7194d66c5304072894a8be3e51e9deda6863c1eceaa35a43d/mypy-2.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3", size = 14188467, upload-time = "2026-08-15T03:02:40.554Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/f4a34edab45667c5465855dc585a20e87978ffa8aee711445b7239d120c6/mypy-2.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f", size = 15225538, upload-time = "2026-08-15T03:03:09.761Z" }, + { url = "https://files.pythonhosted.org/packages/40/05/534b3590757bd05794f73e07f6666c2a77b8597ffed795c94ce570096aa0/mypy-2.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82", size = 15480805, upload-time = "2026-08-15T03:01:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/55/da/bdfba852e2562f599624af5bb7d29e36b0b4f526f2b8bac85efe0dd1803d/mypy-2.3.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9", size = 7761712, upload-time = "2026-08-15T03:02:36.008Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d", size = 11423968, upload-time = "2026-08-15T03:02:06.931Z" }, + { url = "https://files.pythonhosted.org/packages/a9/23/eb5950b24cd26ba3b78f87707a275568d633c77dae8e61c9661be6055ca6/mypy-2.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595", size = 10399323, upload-time = "2026-08-15T03:02:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/82/c7/f80f4e46c0b9a00eb5f78a79d49dda8bdf56a5230f7257fb33e76be04da7/mypy-2.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2", size = 15121308, upload-time = "2026-08-15T03:01:46.053Z" }, + { url = "https://files.pythonhosted.org/packages/5d/74/9b04f17c7074cc5188f02fb63a2ca1d43fedf479e84fe3091c39061a1d7f/mypy-2.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc", size = 15536590, upload-time = "2026-08-15T03:01:35.941Z" }, + { url = "https://files.pythonhosted.org/packages/26/04/c837ef6208e567774e2ed1f863f8ba6ec4817b1b6dd426315e5d559b6ec9/mypy-2.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045", size = 16791074, upload-time = "2026-08-15T03:01:31.073Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/48730230afa45192d5bd429a6a2ff24a6f8dedda90fdf2b221792b54518f/mypy-2.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0", size = 17069183, upload-time = "2026-08-15T03:02:28.566Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/ca23fc9c20eeda09a15c9cbcf50015d0e73f409f6ead059e42aa69a608ff/mypy-2.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63", size = 12154679, upload-time = "2026-08-15T03:02:04.809Z" }, + { url = "https://files.pythonhosted.org/packages/3b/67/8d982126034990869466f73b8db80dcb2234a7ac39b4dad093e047a79835/mypy-2.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4", size = 10969159, upload-time = "2026-08-15T03:02:38.152Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f7/41e7f2d8117fbc7a7587286162ffe2f688984b69c46ed63cf5f2e4fc3bae/mypy-2.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57", size = 13990694, upload-time = "2026-08-15T03:03:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/06/85/8f665811a0c8f3bf6fa1d9acd665ec2d97a2bcc453ae68dcd92340941cd6/mypy-2.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b", size = 14203518, upload-time = "2026-08-15T03:01:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/2d/82/91b866c8546b120bff83b73a439d90d2d63ef3aff113599e6b8e4d566848/mypy-2.3.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561", size = 15220224, upload-time = "2026-08-15T03:01:23.577Z" }, + { url = "https://files.pythonhosted.org/packages/c8/78/c226c99208ee40de7c768369fa533f933afa003dfdc606ff021450724e91/mypy-2.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133", size = 15501512, upload-time = "2026-08-15T03:02:09.453Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e7/7cfb3f106c393979f4cc37ad6c0586044d50401e3c35b0c003e4f3ba6bc9/mypy-2.3.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9", size = 7761913, upload-time = "2026-08-15T03:01:55.65Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/52affefa273b97939a1f474ae4a349c8718635c15b941112dfab4291b0c1/mypy-2.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3", size = 11422533, upload-time = "2026-08-15T03:03:24.101Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b7/75643e70c72a5b346d8a9b1543c967ea8824df2ee3fb7ccba652c272b7bb/mypy-2.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523", size = 10397931, upload-time = "2026-08-15T03:02:55.265Z" }, + { url = "https://files.pythonhosted.org/packages/10/ce/53be21f2d4adfcd26f63f1184a13ed797015ab463853f117e2e11e4d726f/mypy-2.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306", size = 15118669, upload-time = "2026-08-15T03:02:51.479Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/20de757cd42989d291a17fad607742c4c74e875ce5cea00e5a5225020ac1/mypy-2.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021", size = 15545627, upload-time = "2026-08-15T03:03:05.132Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fc/092bdf77ad280eaf501422f0f3b966012b528076cc13e41a774861c907d1/mypy-2.3.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e", size = 16764157, upload-time = "2026-08-15T03:02:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/5c/c94c4d62d909b07f552d0d9356d7acc943825558e602a64822ffa2231536/mypy-2.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc", size = 17073258, upload-time = "2026-08-15T03:02:14.573Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f7/511a88b89e478053c02d22039bb8f3ce4183efe8fd7a4f0a5910a8bb0a32/mypy-2.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4", size = 12135505, upload-time = "2026-08-15T03:02:16.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/bf/02573b56964ecb0f7c644f915f53c325ae15c3faec521c5adf11599a32df/mypy-2.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29", size = 10962647, upload-time = "2026-08-15T03:01:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/41/9675c7a1e78edecfba0b79e587a52594c56e189368261dc7b3a7fffb9527/mypy-2.3.1-py3-none-any.whl", hash = "sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb", size = 2754338, upload-time = "2026-08-15T03:02:53.4Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "networkx" version = "3.4.2" @@ -1780,7 +2055,8 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -1884,7 +2160,8 @@ name = "numpy" version = "2.3.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", @@ -2486,6 +2763,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "platformdirs" version = "4.5.1" From ba469ebe397d27aaaaafb9d038ad44ae54798d24 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 20:51:58 -0400 Subject: [PATCH 12/72] test: pin sweep outcomes, add a copied-storage target and tighten leak checks - Sweep a third registered root C whose storage is a byte copy of A's, so only the workspace_id predicates keep A's rows out; dropping the predicate from _load_record_row now fails the sweep. - Give B and C their own src/ file and run own-argument variants of code_index, code_search, code_todos_scan, code_refactor_propose and workspace_export, so their success responses are scanned. - Pin every call's outcome, including the cross-workspace refusals; the consolidate refusals are rechecked with B's own valid preview token after unlinking, so only the missing link can refuse them. - Scan dictionary keys, forbid the bare canary (except an exact echo of the request), and drop the field-name exemption from the host-path check: no response carries a host path in any field. - Require the core seeds to be real, and report gate-only tools and placeholder seeds next to the swept list. - Pass 2 names the tool when a denial unexpectedly succeeds, requires unchanged A and B ledgers, and matches resource denials exactly with a positive control on A. Co-Authored-By: Claude Opus 5 --- .../test_process_workspace_isolation_sweep.py | 721 ++++++++++++------ 1 file changed, 473 insertions(+), 248 deletions(-) diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 520475d..639e6ee 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -1,22 +1,33 @@ """Workspace isolation and host-path leak sweep across every v7 tool and resource. Pass 1 drives the real production server in-process as the default loopback -principal, which may use both registered workspaces. Every manifest tool and -workspace resource is called against workspace B with workspace A's IDs, -cursors, tokens and idempotency keys. No B response may carry A's seeded text -or IDs, A's database may not change, a rejected call may not change B, and no -response field outside user text may carry a registered root, a storage path -or the home directory. - -Pass 2 relaunches the same workspaces over HTTP with a JWT principal granted -only A. Every call that targets B, directly or as a consolidation or -federation source, must return UNAUTHORIZED_WORKSPACE. +principal, which may use every registered workspace. Every manifest tool and +workspace resource is called against a target workspace with workspace A's +IDs, cursors, tokens and idempotency keys, then again with the target's own +arguments where A's would only be refused. Two targets are swept: + +* B, an ordinary second workspace with its own database; +* C, a registered root whose storage is a byte copy of A's (a re-cloned or + copied project). C's database physically holds A's rows, so only the + ``workspace_id`` predicates keep them out of C's responses. + +No target response may carry A's text, A's canary token, A's IDs or A's +workspace ID; A's database may not change; a rejected call may not change the +target; and no string or key anywhere may carry a registered root or the home +directory. Every call's outcome is pinned. + +Pass 2 relaunches A and B over HTTP with a JWT principal granted only A. +Every call that targets B, directly or as a consolidation or federation +source, must return UNAUTHORIZED_WORKSPACE and write nothing. """ from __future__ import annotations import hashlib +import importlib.util import json +import re +import shutil import sqlite3 from collections.abc import Awaitable, Callable from contextlib import closing @@ -33,6 +44,7 @@ from daem0nmcp.config import Settings from daem0nmcp.covenant import CovenantLevel from daem0nmcp.storage_activation import resolve_active_database +from daem0nmcp.workspace import WorkspaceRegistry from tests.api_v7.process_client import ( call, initialize_workspaces, @@ -44,10 +56,16 @@ ) CANARY = "canary7f3a9d1e" - -# Tools whose purpose is to read or move data between workspaces. Each is -# gated by a directional link (or a preview token that required one) plus an -# authorization check for every workspace involved; see the cross checks below. +APPS_AVAILABLE = importlib.util.find_spec("tree_sitter_language_pack") is not None +PROFILE_ENVIRONMENT = { + "DAEM0NMCP_PROFILE": "core", + "DAEM0NMCP_APPS_ENABLED": "true", + "DAEM0NMCP_GRAPH_ENABLED": "true", +} + +# Tools whose purpose is to read or move data between workspaces. Each needs a +# directional link (or a preview token that required one) plus authorization +# for every workspace involved; see the cross checks below. CROSS_WORKSPACE_TOOLS = ( "workspace_consolidation_preview", "workspace_consolidate", @@ -56,26 +74,9 @@ "workspace_unlink", ) -# Fields that echo text the caller wrote. They are exempt from the host-path -# assertion only (KD-2); A's seeded text and IDs are forbidden in every field. -USER_TEXT_FIELDS = frozenset( - { - "content", - "excerpt", - "bounded_excerpt", - "rationale", - "context", - "outcome_text", - "summary", - "text", - "description", - "label", - "topic", - "trigger", - "recall_query", - "proposed_action", - } -) +# Responses that stop at a gate before any workspace data is read. Tools whose +# every response is one of these are reported separately from the swept list. +GATE_CODES = frozenset({"TASK_REQUIRED", "CAPABILITY_DEGRADED", "CAPABILITY_DISABLED"}) # Derived tables that background projection jobs rewrite after a commit, # including lazily allocated projection-scoped public IDs. @@ -114,24 +115,40 @@ def _ledger_counts(root: Path) -> dict[str, int]: } -def _strings(value: Any, key: str | None = None): +def _strings(value: Any): + """Every string in a JSON value, dictionary keys included.""" if isinstance(value, dict): for child_key, child in value.items(): - yield from _strings(child, child_key) + yield str(child_key) + yield from _strings(child) elif isinstance(value, list): for child in value: - yield from _strings(child, key) + yield from _strings(child) elif isinstance(value, str): - yield key, value + yield value + + +def _invoker(client: Client) -> Invoke: + async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await client.call_tool(tool, arguments, raise_on_error=False) + assert isinstance(result.structured_content, dict), tool + return result.structured_content + + return invoke + + +def _code(response: dict[str, Any]) -> str: + return "ok" if response["ok"] else response["error"]["code"] class Seed: - """Everything workspace A owns that B must never see or resolve.""" + """Everything workspace A owns that another workspace must never see.""" def __init__(self) -> None: self.texts: list[str] = [] self.ids: dict[str, str] = {} self.tokens: dict[str, str] = {} + self.placeholders: set[str] = set() def text(self, label: str) -> str: value = f"{CANARY} {label} owned by workspace A" @@ -139,14 +156,20 @@ def text(self, label: str) -> str: return value def id(self, name: str, prefix: str) -> str: - return self.ids.get(name) or _placeholder(prefix) + if name not in self.ids: + self.placeholders.add(name) + return _placeholder(prefix) + return self.ids[name] def token(self, name: str) -> str: - return self.tokens.get(name) or f"{name}-{CANARY}-placeholder" + if name not in self.tokens: + self.placeholders.add(name) + return f"{name}-{CANARY}-placeholder" + return self.tokens[name] @staticmethod def key(name: str) -> str: - """A's idempotency keys, replayed against B.""" + """A's idempotency keys, replayed against the target.""" return f"sweep-{name}-0001" @@ -171,22 +194,21 @@ async def _protected(invoke: Invoke, workspace_id: str, tool: str, arguments: di ) +async def _target_call(invoke: Invoke, workspace_id: str, tool: str, arguments: dict): + if _is_protected(tool): + return await _protected(invoke, workspace_id, tool, arguments) + return await invoke(tool, {"workspace_id": workspace_id, **arguments}) + + async def _seed_workspace_a(invoke: Invoke, a: str, b: str) -> Seed: + """Write A's objects (and one B record so a consolidation preview exists).""" seed = Seed() - async def ok(tool, arguments, protected=False): - result = await ( - _protected(invoke, a, tool, arguments) - if protected - else invoke(tool, {"workspace_id": a, **arguments}) - ) + async def ok(tool, arguments): + result = await _target_call(invoke, a, tool, arguments) assert result["ok"], f"seed {tool}: {result['error']}" return result["data"] - async def optional(tool, arguments): - result = await invoke(tool, {"workspace_id": a, **arguments}) - return result["data"] if result["ok"] else None - await ok("session_brief", {}) for name, record_type in (("record", "decision"), ("other_record", "warning")): stored = await ok( @@ -198,7 +220,6 @@ async def optional(tool, arguments): "relative_file_path": "src/owned.py", "idempotency_key": seed.key(name), }, - protected=True, ) seed.ids[name] = stored["record"]["record_id"] seed.ids[f"{name}_event"] = stored["event_id"] @@ -220,7 +241,6 @@ async def optional(tool, arguments): "relationship_type": "related_to", "idempotency_key": seed.key("link"), }, - protected=True, ) seed.ids["relationship"] = next( value for value in linked["affected_ids"] if value.startswith("rel_") @@ -232,7 +252,6 @@ async def optional(tool, arguments): "must_do": [seed.text("rule must do")], "idempotency_key": seed.key("rule"), }, - protected=True, ) seed.ids["rule"] = rule["rule_id"] trigger = await ok( @@ -243,33 +262,26 @@ async def optional(tool, arguments): "recall_query": seed.text("trigger recall query"), "idempotency_key": seed.key("trigger"), }, - protected=True, ) seed.ids["trigger"] = trigger["trigger_id"] active = await ok( "active_context_add", {"record_id": seed.ids["record"], "reason": seed.text("active reason")}, - protected=True, ) seed.ids["active_context"] = active["active_context_id"] - page = await ok("active_context_list", {}) - seed.tokens["active_context_selection"] = page["selection_token"] - search = await ok("memory_search_text", {"query": CANARY, "limit": 1}) - seed.tokens["cursor"] = search["next_cursor"] - updates = await ok("session_updates_get", {}) - seed.tokens["session_cursor"] = updates["cursor"] - for tool, arguments in ( - ("memory_prune_preview", {}), - ("memory_duplicates_preview", {}), - ("memory_compaction_preview", {"summary": "sweep", "query": CANARY}), - ("dream_duplicates_preview", {}), - ): - data = await optional(tool, arguments) - if data and data.get("selection_token"): - seed.tokens[tool] = data["selection_token"] - export = await ok("workspace_export", {}) - if export.get("export_session_id"): - seed.ids["export_session"] = export["export_session_id"] + if APPS_AVAILABLE: + await ok("code_index", {"relative_root": "src"}) + found = await ok("code_search", {"query": CANARY}) + seed.ids["code_entity"] = found["items"][0]["code_entity_id"] + # The core graph profile extracts no named entities, so entity and + # community IDs stay placeholders unless an extractor is installed. + await ok("entity_backfill", {"idempotency_key": seed.key("entity_backfill")}) + entities = await ok("entity_list", {}) + if entities["items"]: + seed.ids["entity"] = entities["items"][0]["entity_id"] + communities = await ok("community_list", {}) + if communities["items"]: + seed.ids["community"] = communities["items"][0]["community_id"] # A consolidation preview needs a linked source with at least one record. assert (await invoke("session_brief", {"workspace_id": b}))["ok"] b_record = { @@ -278,26 +290,41 @@ async def optional(tool, arguments): "idempotency_key": "sweep-b-own-record-0001", } assert (await _protected(invoke, b, "memory_store", b_record))["ok"] - await ok("workspace_link", {"linked_workspace_id": b}, protected=True) + await ok("workspace_link", {"linked_workspace_id": b}) + return seed + + +async def _mint_tokens(invoke: Invoke, a: str, b: str, seed: Seed) -> None: + """Cursors and tokens are bound to one server process; mint them per run.""" + + async def ok(tool, arguments): + result = await invoke(tool, {"workspace_id": a, **arguments}) + assert result["ok"], f"mint {tool}: {result['error']}" + return result["data"] + + await ok("session_brief", {}) + seed.tokens["active_context_selection"] = (await ok("active_context_list", {}))[ + "selection_token" + ] + seed.tokens["cursor"] = ( + await ok("memory_search_text", {"query": CANARY, "limit": 1}) + )["next_cursor"] + seed.tokens["session_cursor"] = (await ok("session_updates_get", {}))["cursor"] + for tool, arguments in ( + ("memory_prune_preview", {}), + ("memory_duplicates_preview", {}), + ("memory_compaction_preview", {"summary": "sweep", "query": CANARY}), + ("dream_duplicates_preview", {}), + ): + seed.tokens[tool] = (await ok(tool, arguments))["selection_token"] + export = await ok("workspace_export", {}) + seed.ids["export_session"] = export["export_session_id"] preview = await ok("workspace_consolidation_preview", {"source_workspace_ids": [b]}) seed.tokens["consolidation"] = preview["selection_token"] - # Optional-profile objects: real IDs when the profile is installed. - code = await optional("code_index", {"relative_root": "src"}) - if code is not None: - found = await optional("code_search", {"query": CANARY}) - if found and found["items"]: - seed.ids["code_entity"] = found["items"][0]["code_entity_id"] - entities = await optional("entity_list", {}) - if entities and entities["items"]: - seed.ids["entity"] = entities["items"][0]["entity_id"] - communities = await optional("community_list", {}) - if communities and communities["items"]: - seed.ids["community"] = communities["items"][0]["community_id"] - return seed def _sweep_arguments(seed: Seed) -> dict[str, dict[str, Any]]: - """Arguments for every non-cross tool, aimed at B but naming A's objects.""" + """Arguments for every non-cross tool, naming A's objects.""" record = seed.id("record", "mem") cursor = seed.token("cursor") @@ -359,7 +386,7 @@ def _sweep_arguments(seed: Seed) -> dict[str, dict[str, Any]]: "edit_request_id": _placeholder("edt"), "description": "sweep", }, - "entity_backfill": {"idempotency_key": seed.key("record")}, + "entity_backfill": {"idempotency_key": seed.key("entity_backfill")}, "entity_evolution_trace": {"entity_id": seed.id("entity", "ent")}, "entity_list": {"cursor": cursor}, "knowledge_graph_get": {"record_ids": [record], "query": CANARY}, @@ -418,11 +445,11 @@ def _sweep_arguments(seed: Seed) -> dict[str, dict[str, Any]]: "memory_search_text": {"query": CANARY, "cursor": cursor}, "memory_store": { "record_type": "decision", - "content": "Workspace B sweep decision.", + "content": "Target sweep decision.", "idempotency_key": seed.key("record"), }, "memory_store_batch": { - "records": [{"record_type": "learning", "content": "B batch sweep."}], + "records": [{"record_type": "learning", "content": "Target batch sweep."}], "idempotency_key": seed.key("other_record"), }, "memory_unlink": {"relationship_id": seed.id("relationship", "rel")}, @@ -453,8 +480,146 @@ def _sweep_arguments(seed: Seed) -> dict[str, dict[str, Any]]: } -def _cross_arguments(seed: Seed, source: str) -> dict[str, dict[str, Any]]: - token = seed.token("consolidation") +# The target's own arguments for tools whose A-argument call is only refused, +# so their success responses are scanned too. They run first, so the target +# is indexed before A's code entity is looked up in it. +OWN_VARIANTS: tuple[tuple[str, dict[str, Any]], ...] = ( + ("code_index", {"relative_root": "src"}), + ("code_search", {"query": "neutral"}), + ("code_todos_scan", {"relative_root": "src"}), + ("code_refactor_propose", {"relative_file_path": "src/neutral.py"}), + ("workspace_export", {}), +) + +# Pinned outcomes per tool, in call order: A's arguments, then the variant +# without A's cursor or session (when there is one), then the target's own +# arguments (OWN_VARIANTS). B and the copied-storage C behave identically. +EXPECTED: dict[str, tuple[str, ...]] = { + "active_context_add": ("NOT_FOUND",), + "active_context_clear": ("TOKEN_TAMPERED",), + "active_context_list": ( + "INVALID_ARGUMENT", + "ok", + ), + "active_context_remove": ("NOT_FOUND",), + "code_impact_analyze": ("NOT_FOUND",), + "code_index": ( + "ok", + "ok", + ), + "code_refactor_propose": ( + "WORKSPACE_PATH_ESCAPE", + "ok", + ), + "code_search": ( + "INVALID_ARGUMENT", + "ok", + "ok", + ), + "code_todos_scan": ( + "INVALID_ARGUMENT", + "ok", + "ok", + ), + "code_todos_scan_and_store": ("ok",), + "community_get": ("CAPABILITY_DEGRADED",), + "community_list": ( + "CAPABILITY_DEGRADED", + "CAPABILITY_DEGRADED", + ), + "community_rebuild": ("ok",), + "context_compress": ("ok",), + "context_trigger_create": ("ok",), + "context_trigger_delete": ("NOT_FOUND",), + "context_trigger_list": ( + "INVALID_ARGUMENT", + "ok", + ), + "context_triggers_match": ("ok",), + "covenant_status": ("ok",), + "decision_debate": ("ok",), + "decision_simulate": ("NOT_FOUND",), + "document_ingest_url": ("INVALID_ARGUMENT",), + "dream_duplicates_preview": ("ok",), + "dream_duplicates_purge": ("TOKEN_SCOPE_MISMATCH",), + "edit_preflight": ("NOT_FOUND",), + "entity_backfill": ("ok",), + "entity_evolution_trace": ("NOT_FOUND",), + "entity_list": ( + "INVALID_ARGUMENT", + "ok", + ), + "knowledge_graph_get": ("TASK_REQUIRED",), + "knowledge_graph_render": ("TASK_REQUIRED",), + "knowledge_graph_stats": ("ok",), + "memory_archive_set": ("NOT_FOUND",), + "memory_at_time_get": ("NOT_FOUND",), + "memory_capture_list": ( + "INVALID_ARGUMENT", + "ok", + ), + "memory_capture_promote": ("NOT_FOUND",), + "memory_chain_trace": ("NOT_FOUND",), + "memory_compact": ("TOKEN_SCOPE_MISMATCH",), + "memory_compaction_preview": ("ok",), + "memory_duplicates_cleanup": ("TOKEN_SCOPE_MISMATCH",), + "memory_duplicates_preview": ("ok",), + "memory_link": ("NOT_FOUND",), + "memory_pin_set": ("NOT_FOUND",), + "memory_preflight": ("ok",), + "memory_prune": ("TOKEN_SCOPE_MISMATCH",), + "memory_prune_preview": ("ok",), + "memory_recall": ("ok",), + "memory_recall_entity": ("NOT_FOUND",), + "memory_recall_file": ( + "INVALID_ARGUMENT", + "ok", + ), + "memory_recall_hierarchical": ("ok",), + "memory_record_outcome": ("NOT_FOUND",), + "memory_related": ("NOT_FOUND",), + "memory_search_text": ( + "INVALID_ARGUMENT", + "ok", + ), + "memory_store": ("ok",), + "memory_store_batch": ("ok",), + "memory_unlink": ("NOT_FOUND",), + "memory_verify": ("ok",), + "memory_versions_list": ( + "NOT_FOUND", + "NOT_FOUND", + ), + "projection_rebuild": ("ok",), + "rule_check": ("ok",), + "rule_create": ("ok",), + "rule_evolution_analyze": ("NOT_FOUND",), + "rule_list": ( + "INVALID_ARGUMENT", + "ok", + ), + "rule_update": ("NOT_FOUND",), + "sandbox_execute_python": ("TASK_REQUIRED",), + "session_brief": ("ok",), + "session_updates_get": ( + "INVALID_ARGUMENT", + "ok", + ), + "system_health": ("ok",), + "workspace_export": ( + "IMPORT_INVALID", + "ok", + "ok", + ), + "workspace_import": ("IMPORT_INVALID",), + "workspace_links_list": ( + "INVALID_ARGUMENT", + "ok", + ), +} + + +def _cross_arguments(token: str, source: str) -> dict[str, dict[str, Any]]: return { "workspace_consolidation_preview": {"source_workspace_ids": [source]}, "workspace_consolidate": { @@ -485,192 +650,252 @@ def test_every_manifest_tool_is_swept_or_declared_cross_workspace(): ) -async def test_workspace_isolation_and_path_leak_sweep(tmp_path, capsys): - roots = (tmp_path / "alpha", tmp_path / "beta") - alpha, beta = await initialize_workspaces(roots) - a, b = alpha.workspace_id, beta.workspace_id - source = roots[0] / "src" - source.mkdir() - (source / "owned.py").write_text( - f"def {CANARY}_owned_function():\n return 1 # TODO: {CANARY} owned todo\n", - encoding="utf-8", - ) - settings = Settings( - project_root=str(roots[0]), - workspace_roots=[str(root) for root in roots], - dream_enabled=False, - ) - server = create_v7_server( - "stdio", - settings=settings, - environ={ - "DAEM0NMCP_PROFILE": "core", - "DAEM0NMCP_APPS_ENABLED": "true", - "DAEM0NMCP_GRAPH_ENABLED": "true", - }, - ) - path_needles = { - str(path).lower() - for base in (*roots, Path.home()) - for path in (base, base.resolve()) - } - path_needles |= {str(root / ".daem0nmcp" / "storage").lower() for root in roots} - path_needles |= {Path(needle).as_posix().lower() for needle in path_needles} - - swept: list[str] = [] - async with Client(server) as client: +class Sweep: + """Pass-1 assertions shared by every target workspace.""" - async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: - result = await client.call_tool(tool, arguments, raise_on_error=False) - assert isinstance(result.structured_content, dict), tool - return result.structured_content - - served = {tool.name for tool in await client.list_tools()} - assert served == set(V7_TOOL_LEVELS) - seed = await _seed_workspace_a(invoke, a, b) - forbidden = [*seed.texts, *seed.ids.values(), a] - a_ledger = _ledger_counts(roots[0]) - - def assert_isolated(label: str, response: Any) -> None: - for key, value in _strings(response): - for secret in forbidden: - assert secret not in value, f"{label}: A's {secret!r} in {key}" - if key in USER_TEXT_FIELDS: - continue - lowered = value.lower() - for needle in path_needles: - assert needle not in lowered, f"{label}: host path in {key}" - assert _ledger_counts(roots[0]) == a_ledger, f"{label} changed A" - - assert (await invoke("session_brief", {"workspace_id": b}))["ok"] - for tool, arguments in sorted(_sweep_arguments(seed).items()): + def __init__(self, roots: dict[str, Path], a_root: Path, seed: Seed, a: str): + self.a_root = a_root + self.seed = seed + self.forbidden = [*seed.texts, *seed.ids.values(), a] + self.a_ledger = _ledger_counts(a_root) + needles = { + str(path).lower() + for base in (*roots.values(), Path.home()) + for path in (base, base.resolve()) + } + self.needles = needles | {Path(n).as_posix().lower() for n in needles} + self.observed: dict[str, dict[str, list[str]]] = {} + + def check(self, label: str, response: Any, request: Any) -> None: + echoed = set(_strings(request)) + for value in _strings(response): + for secret in self.forbidden: + assert secret not in value, f"{label}: A's {secret!r} leaked" + # A bare canary may appear only as an exact echo of the request + # (memory_verify returns its claim). + assert CANARY not in value or value in echoed, f"{label}: {value!r}" + lowered = value.lower() + for needle in self.needles: + assert needle not in lowered, f"{label}: host path in {value!r}" + assert _ledger_counts(self.a_root) == self.a_ledger, f"{label} changed A" + + async def run(self, client, name: str, target: str, root: Path) -> None: + invoke = _invoker(client) + observed = self.observed.setdefault(name, {}) + assert (await invoke("session_brief", {"workspace_id": target}))["ok"] + own: dict[str, str] = {} + for tool, arguments in OWN_VARIANTS: + response = await _target_call(invoke, target, tool, arguments) + self.check(f"{name}:{tool} (own)", response, arguments) + own[tool] = _code(response) + for tool, arguments in sorted(_sweep_arguments(self.seed).items()): variants = [arguments] - if {"cursor", "after_cursor"} & set(arguments): - # A's cursor is refused, so also read B's own first page. + if {"cursor", "after_cursor", "export_session_id"} & set(arguments): + # A's cursor or session is refused; also read the target's own. variants.append( { key: value for key, value in arguments.items() - if key not in {"cursor", "after_cursor"} + if key not in {"cursor", "after_cursor", "export_session_id"} } ) for variant in variants: - before = _ledger_counts(roots[1]) - if _is_protected(tool): - response = await _protected(invoke, b, tool, variant) - else: - response = await invoke(tool, {"workspace_id": b, **variant}) - assert_isolated(tool, response) + before = _ledger_counts(root) + response = await _target_call(invoke, target, tool, variant) + self.check(f"{name}:{tool}", response, variant) if not response["ok"]: - assert _ledger_counts(roots[1]) == before, ( - f"rejected {tool} wrote B" - ) - swept.append(tool) - - templates = await client.list_resource_templates() - workspace_templates = [ - template.uriTemplate - for template in templates - if template.uriTemplate.startswith("memory://workspaces/{workspace_id}/") + assert _ledger_counts(root) == before, f"{name}:{tool} wrote" + observed.setdefault(tool, []).append(_code(response)) + for tool, code in own.items(): + observed[tool].append(code) + templates = [ + f"memory://workspaces/{target}/{kind}" + for kind in ("warnings", "failures", "rules", "active-context") ] - assert workspace_templates - for template in workspace_templates: - uri = template.replace("{workspace_id}", b) + for uri in templates: contents = await client.read_resource(uri) - assert_isolated(uri, [json.loads(content.text) for content in contents]) - swept.append(uri) - - # Cross-workspace tools: B has no link to A (only A -> B exists), so - # every B-side read or move of A must be refused; a federated recall is - # refused the same way. - federated = await invoke( - "memory_recall", - {"workspace_id": b, "query": CANARY, "linked_workspace_ids": [a]}, - ) - assert not federated["ok"] - assert_isolated("memory_recall linked", federated) - for tool, arguments in _cross_arguments(seed, a).items(): - if tool in {"workspace_link", "workspace_unlink"}: - continue - before = _ledger_counts(roots[1]) - if _is_protected(tool): - response = await _protected(invoke, b, tool, arguments) - else: - response = await invoke(tool, {"workspace_id": b, **arguments}) - assert not response["ok"], f"{tool} crossed without a link" - assert_isolated(tool, response) - assert _ledger_counts(roots[1]) == before - # Linking needs a preflight bound to the exact target, then the - # direction it names is what consolidation honours. - unbound = await invoke( - "workspace_link", - { - "workspace_id": b, - "linked_workspace_id": a, - "preflight_token": seed.token("consolidation"), - }, + self.check(uri, [json.loads(item.text) for item in contents], uri) + + def assert_pinned(self, name: str) -> None: + for tool, codes in sorted(self.observed[name].items()): + if APPS_AVAILABLE or not tool.startswith("code_"): + assert tuple(codes) == EXPECTED[tool], (name, tool, codes) + + def gated(self, name: str) -> list[str]: + return sorted( + tool + for tool, codes in self.observed[name].items() + if all(code in GATE_CODES for code in codes) ) - assert not unbound["ok"] - link = _cross_arguments(seed, a)["workspace_link"] - assert (await _protected(invoke, b, "workspace_link", link))["ok"] - preview = await invoke( - "workspace_consolidation_preview", - {"workspace_id": b, "source_workspace_ids": [a]}, + + +async def test_workspace_isolation_and_path_leak_sweep(tmp_path, capsys): + a_root, b_root, c_root = (tmp_path / name for name in ("alpha", "beta", "gamma")) + alpha, beta = await initialize_workspaces((a_root, b_root)) + a, b = alpha.workspace_id, beta.workspace_id + c = WorkspaceRegistry([c_root], default_root=c_root).default.workspace_id + for root, text in ( + (a_root, f"def {CANARY}_owned_function():\n return 1 # TODO: {CANARY}\n"), + (b_root, "def neutral_function():\n return 1 # TODO: neutral\n"), + ): + (root / "src").mkdir() + (root / "src" / f"{'owned' if root == a_root else 'neutral'}.py").write_text( + text, encoding="utf-8" ) - assert preview["ok"], preview["error"] - unlink = _cross_arguments(seed, a)["workspace_unlink"] - assert (await _protected(invoke, b, "workspace_unlink", unlink))["ok"] - after_unlink = await invoke( - "workspace_consolidation_preview", - {"workspace_id": b, "source_workspace_ids": [a]}, + + def server(roots): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(a_root), + workspace_roots=[str(root) for root in roots], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, ) - assert not after_unlink["ok"] - # Pass 2: a JWT principal granted only A is refused B everywhere. + # Pass 1, target B. + async with Client(server((a_root, b_root))) as client: + invoke = _invoker(client) + assert {tool.name for tool in await client.list_tools()} == set(V7_TOOL_LEVELS) + seed = await _seed_workspace_a(invoke, a, b) + await _mint_tokens(invoke, a, b, seed) + required = {"record", "other_record", "relationship", "rule", "trigger"} + required |= {"active_context"} + required |= {"export_session"} | ({"code_entity"} if APPS_AVAILABLE else set()) + assert required <= set(seed.ids), required - set(seed.ids) + roots = {"A": a_root, "B": b_root, "C": c_root} + sweep = Sweep(roots, a_root, seed, a) + await sweep.run(client, "B", b, b_root) + cross = await _cross_workspace_checks(invoke, sweep, a, b, b_root, seed) + + # Pass 1, target C: a registered root holding a byte copy of A's storage. + shutil.copytree(a_root / ".daem0nmcp", c_root / ".daem0nmcp") + (c_root / "src").mkdir() + (c_root / "src" / "neutral.py").write_text( + "def neutral_function():\n return 1 # TODO: neutral\n", encoding="utf-8" + ) + async with Client(server((a_root, b_root, c_root))) as client: + invoke = _invoker(client) + await _mint_tokens(invoke, a, b, seed) + sweep.a_ledger = _ledger_counts(a_root) + await sweep.run(client, "C", c, c_root) + for name in ("B", "C"): + sweep.assert_pinned(name) + + await _jwt_pass(tmp_path, a_root, b_root, a, b, seed) + + with capsys.disabled(): + swept = sorted(sweep.observed["B"]) + print(f"\nswept ({len(swept)} tools + 4 resources, targets B and C)") + print(f"gate-only on B (no data path reached): {sweep.gated('B')}") + print(f"placeholder seeds (no real A object): {sorted(seed.placeholders)}") + print(f"cross-workspace ({len(cross)}): {', '.join(cross)}") + + +async def _cross_workspace_checks(invoke, sweep, a, b, b_root, seed) -> list[str]: + """B may read or move A only through a B -> A link it created itself.""" + + async def refused(tool, arguments, code): + before = _ledger_counts(b_root) + response = await _target_call(invoke, b, tool, arguments) + sweep.check(f"B:{tool}", response, arguments) + assert _code(response) == code, (tool, response) + assert _ledger_counts(b_root) == before, tool + + # Only A -> B exists, so every B-side read or move of A is refused. + await refused( + "memory_recall", + {"query": CANARY, "linked_workspace_ids": [a]}, + "UNAUTHORIZED_WORKSPACE", + ) + for tool, arguments in _cross_arguments(seed.token("consolidation"), a).items(): + if tool in {"workspace_link", "workspace_unlink"}: + continue + await refused(tool, arguments, "UNAUTHORIZED_WORKSPACE") + # Linking needs a preflight bound to exactly that link. + unbound = await invoke( + "workspace_link", + { + "workspace_id": b, + "linked_workspace_id": a, + "preflight_token": seed.token("consolidation"), + }, + ) + assert _code(unbound) == "TOKEN_TAMPERED", unbound + link = {"linked_workspace_id": a} + assert (await _protected(invoke, b, "workspace_link", link))["ok"] + preview = await invoke( + "workspace_consolidation_preview", + {"workspace_id": b, "source_workspace_ids": [a]}, + ) + assert preview["ok"], preview["error"] + token = preview["data"]["selection_token"] + assert (await _protected(invoke, b, "workspace_unlink", link))["ok"] + # With B's own valid preview token, only the removed link refuses the move. + for tool, arguments in _cross_arguments(token, a).items(): + if tool in {"workspace_link", "workspace_unlink"}: + continue + await refused(tool, arguments, "UNAUTHORIZED_WORKSPACE") + return list(CROSS_WORKSPACE_TOOLS) + + +async def _jwt_pass(tmp_path, a_root, b_root, a, b, seed) -> None: + """Pass 2: a JWT principal granted only A is refused B everywhere.""" policy_path = tmp_path / "protected" / "access.json" write_workspace_grants(policy_path, [a]) + placeholder_token = f"preflight-{CANARY}-placeholder" with jwt_issuer() as (issuer_url, token): async with process_client( - roots[0], + a_root, "streamable-http", - workspace_roots=roots, + workspace_roots=(a_root, b_root), environment_overrides=jwt_environment(issuer_url, policy_path), http_headers={"Authorization": "Bearer " + token()}, ) as session: + await succeed(session, "session_brief", {"workspace_id": a}) + ledgers = {root: _ledger_counts(root) for root in (a_root, b_root)} - async def remote(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: - return await call(session, tool, arguments) + def assert_denied(tool, denied): + assert not denied["ok"], (tool, "succeeded for an ungranted workspace") + assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE", ( + tool, + denied, + ) - await succeed(session, "session_brief", {"workspace_id": a}) - placeholder_token = f"preflight-{CANARY}-placeholder" targets = { **_sweep_arguments(seed), - **_cross_arguments(seed, a), + **_cross_arguments(seed.token("consolidation"), a), } for tool, arguments in sorted(targets.items()): if _is_protected(tool): arguments = {**arguments, "preflight_token": placeholder_token} - denied = await remote(tool, {"workspace_id": b, **arguments}) - assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE", tool - for template in workspace_templates: - with pytest.raises(McpError): - await session.read_resource(template.replace("{workspace_id}", b)) - # B as a source of an A-side federated read or consolidation. - denied = await remote( + denied = await call(session, tool, {"workspace_id": b, **arguments}) + assert_denied(tool, denied) + for kind in ("warnings", "failures", "rules", "active-context"): + # The server masks the reason, so the same read succeeding for + # A is what shows B's failure is the authorization refusal. + await session.read_resource(f"memory://workspaces/{a}/{kind}") + uri = f"memory://workspaces/{b}/{kind}" + with pytest.raises( + McpError, match=f"^Error reading resource '{re.escape(uri)}'$" + ): + await session.read_resource(uri) + # B as the source of an A-side federated read or consolidation. + denied = await call( + session, "memory_recall", {"workspace_id": a, "query": CANARY, "linked_workspace_ids": [b]}, ) - assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE" - for tool, arguments in _cross_arguments(seed, b).items(): - if _is_protected(tool): - denied = await _protected(remote, a, tool, arguments) - else: - denied = await remote(tool, {"workspace_id": a, **arguments}) - assert denied["error"]["code"] == "UNAUTHORIZED_WORKSPACE", tool + assert_denied("memory_recall linked", denied) - with capsys.disabled(): - print(f"\nswept ({len(swept)}): {', '.join(swept)}") - print( - f"cross-workspace ({len(CROSS_WORKSPACE_TOOLS)}): " - + ", ".join(CROSS_WORKSPACE_TOOLS) - ) + async def remote(tool, arguments): + return await call(session, tool, arguments) + + for tool, arguments in _cross_arguments( + seed.token("consolidation"), b + ).items(): + denied = await _target_call(remote, a, tool, arguments) + assert_denied(f"{tool} from B", denied) + for root, counts in ledgers.items(): + assert _ledger_counts(root) == counts, f"denied calls wrote {root.name}" From a0ca83a629de8daebc6674b8908b59ebf12f50cb Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:17:33 -0400 Subject: [PATCH 13/72] fix: take ownership only from BUILTIN\Administrators Ownership is reassigned to the current user only when a new protected path is owned by BUILTIN\Administrators (an elevated administrator's default); any other foreign owner is still refused by verification. Unit tests replay both owners without needing elevation. Co-Authored-By: Claude Opus 5 --- daem0nmcp/protected_files.py | 16 ++++++++----- tests/test_protected_files.py | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/daem0nmcp/protected_files.py b/daem0nmcp/protected_files.py index c5fdce9..29d905b 100644 --- a/daem0nmcp/protected_files.py +++ b/daem0nmcp/protected_files.py @@ -264,13 +264,17 @@ def _protect_windows(path: Path, *, directory: bool) -> None: ): raise _win_error("setting protected path security") # An elevated administrator's new files are owned by - # BUILTIN\Administrators. The owner-only DACL just applied grants - # this user WRITE_OWNER, so take ownership explicitly. + # BUILTIN\Administrators (BA). The owner-only DACL just applied + # grants this user WRITE_OWNER, so take ownership from BA only; + # any other foreign owner is refused by the check below. + actual = _actual_sddl(path) owner = expected.split("D:", 1)[0] + "D:" - if not _actual_sddl(path).startswith( - owner - ) and not _ADVAPI32.SetFileSecurityW( - str(path), _OWNER_SECURITY_INFORMATION, descriptor + if ( + not actual.startswith(owner) + and actual.startswith("O:BAD:") + and not _ADVAPI32.SetFileSecurityW( + str(path), _OWNER_SECURITY_INFORMATION, descriptor + ) ): raise _win_error("setting protected path owner") finally: diff --git a/tests/test_protected_files.py b/tests/test_protected_files.py index 1d97ba3..54ff80f 100644 --- a/tests/test_protected_files.py +++ b/tests/test_protected_files.py @@ -56,6 +56,51 @@ def test_windows_rejects_extra_users_access(tmp_path): verify_owner_only_file(target) +def _fake_windows_security(monkeypatch, owners): + """Replay owner SIDs ("USER" = current user) and record SetFileSecurityW.""" + from daem0nmcp import protected_files + + expected_descriptor, expected = protected_files._converted_sddl( + protected_files._desired_sddl(directory=False) + ) + protected_files._KERNEL32.LocalFree(expected_descriptor) + dacl = expected.split("D:", 1)[1] + user = expected.split("D:", 1)[0][2:] + sequence = iter( + f"O:{user if owner == 'USER' else owner}D:{dacl}" for owner in owners + ) + calls = [] + monkeypatch.setattr(protected_files, "_actual_sddl", lambda _path: next(sequence)) + monkeypatch.setattr( + protected_files._ADVAPI32, + "SetFileSecurityW", + lambda _path, information, _descriptor: calls.append(information) or True, + ) + return protected_files, calls + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows owner test") +def test_windows_takes_ownership_only_from_builtin_administrators( + tmp_path, monkeypatch +): + target = tmp_path / "file" + target.write_bytes(b"") + module, calls = _fake_windows_security(monkeypatch, ["BA", "USER"]) + module._protect_windows(target, directory=False) + assert calls[-1] == module._OWNER_SECURITY_INFORMATION + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows owner test") +def test_windows_refuses_any_other_foreign_owner(tmp_path, monkeypatch): + target = tmp_path / "file" + target.write_bytes(b"") + foreign = "S-1-5-21-1-2-3-1001" + module, calls = _fake_windows_security(monkeypatch, [foreign, foreign]) + with pytest.raises(ProtectedPathError, match="verification failed"): + module._protect_windows(target, directory=False) + assert module._OWNER_SECURITY_INFORMATION not in calls + + def test_rejects_linked_ancestry(tmp_path): real = tmp_path / "real" real.mkdir() From e65b4013624dd887f153cd2744540b4881cf0c33 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:18:18 -0400 Subject: [PATCH 14/72] fix: never read a migration source as immutable next to a rollback journal A -journal file beside the source may be a hot journal that SQLite must see, so it now selects the ordinary mode=ro path like -wal and -shm do. Co-Authored-By: Claude Opus 5 --- daem0nmcp/migrations/v7.py | 12 +++++++----- tests/test_migrate_v7.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index ce778ca..ac55d19 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -149,14 +149,16 @@ def _readonly_connection(path: Path) -> sqlite3.Connection: # Migration dry-run must not write any byte. On Windows the no-lock VFS # keeps read-only connections off the ``-shm`` read marks. POSIX has no # equivalent: ``unix-none`` lacks shared memory, so it cannot open a WAL - # database at all. There, with no ``-wal``/``-shm`` the main file is the - # whole database and is read as immutable (SQLite would otherwise create - # both files); a live WAL is read through the default VFS, whose only - # write is the volatile ``-shm`` wal-index. + # database at all. There, with no ``-wal``, ``-shm`` or ``-journal`` the + # main file is the whole database and is read as immutable (SQLite would + # otherwise create the WAL files); anything else is read through the + # default VFS, whose only write is the volatile ``-shm`` wal-index. resolved = path.resolve() if sys.platform == "win32": parameters = "&vfs=win32-none" - elif not any(Path(f"{resolved}{suffix}").exists() for suffix in ("-wal", "-shm")): + elif not any( + Path(f"{resolved}{suffix}").exists() for suffix in ("-wal", "-shm", "-journal") + ): parameters = "&immutable=1" else: parameters = "" diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index 271069a..be21454 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -6,6 +6,7 @@ import json import shutil import sqlite3 +import sys import tempfile import unittest from pathlib import Path @@ -180,6 +181,32 @@ def test_dry_run_is_deterministic_wal_aware_and_strictly_read_only(self): finally: writer.close() + @unittest.skipIf(sys.platform == "win32", "Windows always uses win32-none") + def test_sidecar_files_keep_the_source_off_the_immutable_path(self): + from daem0nmcp.migrations import v7 + + with tempfile.TemporaryDirectory() as raw: + database = Path(raw) / "daem0nmcp.db" + _create_legacy_database(database).close() + uris = [] + connect = sqlite3.connect + + def recording_connect(uri, **kwargs): + uris.append(uri) + return connect(uri, **kwargs) + + with mock.patch.object(v7.sqlite3, "connect", recording_connect): + v7._readonly_connection(database).close() + for suffix in ("-wal", "-shm", "-journal"): + sidecar = Path(f"{database}{suffix}") + sidecar.write_bytes(b"") + try: + v7._readonly_connection(database).close() + finally: + sidecar.unlink(missing_ok=True) + self.assertIn("immutable=1", uris[0]) + self.assertTrue(all("immutable" not in uri for uri in uris[1:])) + def test_dry_run_resolves_only_registered_workspace(self): from daem0nmcp.migrations.v7 import MigrationV7Service From 44bd8a92767d7626e628ed17231056f54e79e239 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:19:52 -0400 Subject: [PATCH 15/72] test: require the own-argument code search to return results Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_process_workspace_isolation_sweep.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 639e6ee..93ea35a 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -688,6 +688,9 @@ async def run(self, client, name: str, target: str, root: Path) -> None: response = await _target_call(invoke, target, tool, arguments) self.check(f"{name}:{tool} (own)", response, arguments) own[tool] = _code(response) + if tool == "code_search" and APPS_AVAILABLE: + # The scan must cover real results, not an empty page. + assert response["data"]["items"], (name, response) for tool, arguments in sorted(_sweep_arguments(self.seed).items()): variants = [arguments] if {"cursor", "after_cursor", "export_session_id"} & set(arguments): From ad8f8a080c6acb453531c1aa67f2a73f66a906fb Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:24:42 -0400 Subject: [PATCH 16/72] test: name the recording connect parameter so uri=True binds on POSIX Co-Authored-By: Claude Opus 5 --- tests/test_migrate_v7.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index be21454..cb4969d 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -191,9 +191,9 @@ def test_sidecar_files_keep_the_source_off_the_immutable_path(self): uris = [] connect = sqlite3.connect - def recording_connect(uri, **kwargs): - uris.append(uri) - return connect(uri, **kwargs) + def recording_connect(database, **kwargs): + uris.append(database) + return connect(database, **kwargs) with mock.patch.object(v7.sqlite3, "connect", recording_connect): v7._readonly_connection(database).close() From 57b2e3ed09d93399366574c61cd07d41d4841697 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:28:28 -0400 Subject: [PATCH 17/72] test: pin a supported Python for the models-local readiness tests models-local needs Python 3.11+ (its extra installs nothing on 3.10), so on 3.10 the registry correctly reports upgrade_python before probing packages. The missing-extra and ready-probe tests now pin 3.11 like the python-floor test pins 3.10. Co-Authored-By: Claude Opus 5 --- tests/test_capabilities.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index aabe7d1..8459d84 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -169,7 +169,10 @@ def test_missing_extra_is_degraded_with_install_remediation(self): module_available=lambda _: False, ) - capability = registry.get("models-local") + # models-local needs Python 3.11+ (its extra installs nothing on 3.10, + # see test_models_local_reports_python_floor); pin a supported version. + with patch("daem0nmcp.capabilities.sys.version_info", (3, 11, 0)): + capability = registry.get("models-local") self.assertEqual(capability["status"], "degraded") self.assertEqual(capability["remediation"]["action"], "install_extra") @@ -242,7 +245,8 @@ def test_ready_profile_checks_only_the_requested_profile(self): module_available=lambda module: probes.append(module) or True, ) - capability = registry.get("models-local") + with patch("daem0nmcp.capabilities.sys.version_info", (3, 11, 0)): + capability = registry.get("models-local") self.assertEqual(capability["status"], "ready") self.assertEqual( From 76e46415f26e7ca70eafd19f79bf938a9bd11c19 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:28:28 -0400 Subject: [PATCH 18/72] test: wait for the post-lock lease renewal instead of a fixed sleep The competitor ran a fixed 80 ms after the builder released the writer lock, so a slow runner could let the 100 ms lease lapse before the heartbeat renewed it. The test now polls for a lease renewed after the commit (5 s deadline) and uses a 500 ms lease with a 750 ms lock hold, so the lock still outlasts the lease as before (F-092). Co-Authored-By: Claude Opus 5 --- tests/test_retrieval_jobs.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_retrieval_jobs.py b/tests/test_retrieval_jobs.py index be51995..9bda81e 100644 --- a/tests/test_retrieval_jobs.py +++ b/tests/test_retrieval_jobs.py @@ -331,9 +331,21 @@ def write_locked_build(_workspace_id): self.connection.execute( "UPDATE background_jobs SET result_json=result_json" ) - time.sleep(0.25) + # Hold the writer lock past the whole lease, so a renewal computed + # from a pre-lock clock sample would already be expired. + time.sleep(0.75) self.connection.commit() - time.sleep(0.08) + committed_us = time.time_ns() // 1_000 + # Wait for the heartbeat's post-lock renewal instead of a fixed + # sleep; slow runners may take longer than one interval. + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + (expires_us,) = self.connection.execute( + "SELECT lease_expires_at_us FROM background_jobs" + ).fetchone() + if expires_us > committed_us: + break + time.sleep(0.01) competitor_connection = sqlite3.connect(self.database_path) competitor_connection.row_factory = sqlite3.Row try: @@ -343,8 +355,8 @@ def write_locked_build(_workspace_id): clock_us=lambda: time.time_ns() // 1_000, lease_owner="competing-worker", token_factory=lambda: "competing-token", - lease_duration_us=100_000, - heartbeat_interval_us=20_000, + lease_duration_us=500_000, + heartbeat_interval_us=100_000, retry_delay_us=1, ) competing_results.append(competitor.run_once()) @@ -357,8 +369,8 @@ def write_locked_build(_workspace_id): clock_us=lambda: time.time_ns() // 1_000, lease_owner="test-worker", token_factory=lambda: "lease-token", - lease_duration_us=100_000, - heartbeat_interval_us=20_000, + lease_duration_us=500_000, + heartbeat_interval_us=100_000, retry_delay_us=1, ) From f7f024aec79b8013764ed0541f21608ba637792c Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 21:30:32 -0400 Subject: [PATCH 19/72] test: poll for the dreaming graph refresh to finish instead of sleeping The graph-generation test read system_health after a fixed 2 s sleep, so a slow runner could still see community_refresh running. It now polls every 250 ms (30 s deadline) until the refresh is no longer running and has succeeded, then makes the same assertions (F-092). Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_process_dreaming.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/api_v7/test_process_dreaming.py b/tests/api_v7/test_process_dreaming.py index dbe4321..8954f36 100644 --- a/tests/api_v7/test_process_dreaming.py +++ b/tests/api_v7/test_process_dreaming.py @@ -194,11 +194,19 @@ async def test_production_dreaming_builds_real_graph_generation( environment_overrides=environment, ) as session: await succeed(session, "session_brief", scope) - await asyncio.sleep(2.0) - health = await succeed(session, "system_health", scope) - strategies = { - item["strategy"]: item for item in health["dreaming"]["strategies"] - } + # Poll until the refresh has finished instead of sleeping a fixed time; + # slow runners can still be mid-run after two seconds. + deadline = asyncio.get_running_loop().time() + 30 + while True: + await asyncio.sleep(0.25) + health = await succeed(session, "system_health", scope) + strategies = { + item["strategy"]: item for item in health["dreaming"]["strategies"] + } + refresh = strategies["community_refresh"] + done = refresh["status"] != "running" and refresh["last_success_at"] + if done or asyncio.get_running_loop().time() >= deadline: + break assert strategies["community_refresh"]["status"] == "idle" assert strategies["community_refresh"]["last_success_at"] is not None with sqlite3.connect(database) as connection: From 220f98a8b8ba3c6d51d7a3c88379bb9a5a048f60 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:00:56 -0400 Subject: [PATCH 20/72] fix: make the Claude Code hooks remind instead of block pre_edit is now stdlib-only: outside a .daem0nmcp project it exits 0 silently; inside one it adds a one-line additionalContext reminder to call memory_recall_file and memory_preflight, and never sets permissionDecision. The stop hook reads cwd, transcript_path and session_id from its stdin event, unwraps Claude Code's nested transcript records, and shows its suggestions as a systemMessage; it never emits "decision" or exits 2. post_edit only imports the edit bridge when the project is paired. uninstall-claude-hooks with --project-path strips the pairing env keys from the project's settings.local.json; --remove-credentials also deletes the local authority credential and socket directories. The manual hooks/ scripts become exit-0 deprecation stubs and settings.json.example is gone. The bridge integration tests now drive the bridge through a test-only host driver instead of the deleted handle_pre_edit. Co-Authored-By: Claude Opus 5 --- daem0nmcp/claude_hooks/_client.py | 25 +-- daem0nmcp/claude_hooks/install.py | 105 +++++++++-- daem0nmcp/claude_hooks/post_edit.py | 18 +- daem0nmcp/claude_hooks/pre_edit.py | 196 ++++++--------------- daem0nmcp/claude_hooks/stop.py | 56 +++--- daem0nmcp/cli.py | 11 +- daem0nmcp/edit_host.py | 35 ++-- hooks/daem0n_post_edit_hook.py | 142 +-------------- hooks/daem0n_pre_edit_hook.py | 102 +---------- hooks/daem0n_prompt_hook.py | 52 +----- hooks/daem0n_stop_hook.py | 150 +--------------- hooks/settings.json.example | 70 -------- tests/api_v7/test_docs_hooks.py | 164 +++++------------- tests/api_v7/test_process_edit_capture.py | 10 +- tests/native_edit_host.py | 121 +++++++++++++ tests/test_claude_hooks_client.py | 22 +-- tests/test_claude_hooks_install.py | 96 ++++++++++ tests/test_claude_hooks_post_edit.py | 137 ++++++--------- tests/test_claude_hooks_pre_edit.py | 202 ++++++++++++---------- tests/test_claude_hooks_stop.py | 92 ++++++++++ tests/test_edit_bridge_transport.py | 10 +- tests/test_edit_host.py | 56 +++--- 22 files changed, 804 insertions(+), 1068 deletions(-) delete mode 100644 hooks/settings.json.example create mode 100644 tests/native_edit_host.py diff --git a/daem0nmcp/claude_hooks/_client.py b/daem0nmcp/claude_hooks/_client.py index fcd1f2d..53fd902 100644 --- a/daem0nmcp/claude_hooks/_client.py +++ b/daem0nmcp/claude_hooks/_client.py @@ -5,7 +5,6 @@ Daem0n-MCP's database, memory, and rules from hook scripts. """ -import asyncio import json import os import sys @@ -75,28 +74,18 @@ def get_managers(project_path: str): def run_async(coro) -> Any: """Run an async coroutine synchronously.""" + import asyncio # lazy: keeps the stdlib-only pre_edit hook fast + return asyncio.run(coro) -def get_tool_input() -> dict: - """Parse TOOL_INPUT env var as JSON, returning {} on failure.""" - raw = os.environ.get("TOOL_INPUT", "{}") +def get_command_from_input() -> str | None: + """Extract command from the legacy TOOL_INPUT env var (for Bash hooks).""" try: - return json.loads(raw) + data = json.loads(os.environ.get("TOOL_INPUT", "{}")) except (json.JSONDecodeError, TypeError): - return {} - - -def get_file_path_from_input() -> str | None: - """Extract file_path or notebook_path from tool input.""" - data = get_tool_input() - return data.get("file_path") or data.get("notebook_path") - - -def get_command_from_input() -> str | None: - """Extract command from tool input (for Bash hooks).""" - data = get_tool_input() - return data.get("command") + return None + return data.get("command") if isinstance(data, dict) else None def block(message: str) -> NoReturn: diff --git a/daem0nmcp/claude_hooks/install.py b/daem0nmcp/claude_hooks/install.py index cbfa4a6..cd60be0 100644 --- a/daem0nmcp/claude_hooks/install.py +++ b/daem0nmcp/claude_hooks/install.py @@ -7,11 +7,25 @@ import argparse import json +import shutil import sys from pathlib import Path from typing import Any -from ..edit_host import provision_client_bridge_installation +from ..edit_host import ( + local_bridge_layout, + provision_client_bridge_installation, +) + +# Every env key pairing can write into /.claude/settings.local.json. +_BRIDGE_ENV_KEYS = ( + "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", + "DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR", + "DAEM0NMCP_EDIT_BRIDGE_MODE", + "DAEM0NMCP_EDIT_HOST_WORKSPACE_BINDING_FILE", + "DAEM0NMCP_PROJECT_ROOT", + "DAEM0NMCP_STORAGE_PATH", +) def _settings_path() -> Path: @@ -236,11 +250,41 @@ def install_claude_hooks( return True, message -def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: +def _unconfigure_project_bridge(project_root: Path, dry_run: bool) -> Path | None: + """Strip the env keys pairing added; return the settings path if changed.""" + path = project_root / ".claude" / "settings.local.json" + if not path.is_file(): + return None + value = json.loads(path.read_text(encoding="utf-8")) + configured = value.get("env") if isinstance(value, dict) else None + if not isinstance(configured, dict): + return None + removed = [key for key in _BRIDGE_ENV_KEYS if key in configured] + if not removed: + return None + for key in removed: + del configured[key] + if not configured: + del value["env"] + if not dry_run: + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return path + + +def uninstall_claude_hooks( + dry_run: bool = False, + *, + project_path: str | Path | None = None, + remove_credentials: bool = False, + bridge_config_root: Path | None = None, +) -> tuple[bool, str]: """ Remove all Daem0n Claude Code hooks. - Preserves all other hooks. Cleans up empty event lists. + Preserves all other hooks. Cleans up empty event lists. With + *project_path*, also strips the pairing env keys from that project's + ``.claude/settings.local.json``; with *remove_credentials*, also deletes + the project's local edit-bridge credential and socket directories. Returns (success, message). """ @@ -258,21 +302,43 @@ def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: else: del hooks_section[event] - if not removed_events: - return True, "No Daem0n hooks found to remove." - - settings["hooks"] = hooks_section - - if dry_run: - formatted = json.dumps(settings, indent=2) - return True, f"[dry-run] Would write to {_settings_path()}:\n{formatted}" + lines: list[str] = [] + if removed_events: + settings["hooks"] = hooks_section + if dry_run: + formatted = json.dumps(settings, indent=2) + lines.append(f"[dry-run] Would write to {_settings_path()}:\n{formatted}") + else: + try: + _write_settings(settings) + except OSError as exc: + return False, f"Failed to write settings: {exc}" + lines.append( + f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}" + ) + else: + lines.append("No Daem0n hooks found to remove.") - try: - _write_settings(settings) - except OSError as exc: - return False, f"Failed to write settings: {exc}" + if project_path is not None: + prefix = "[dry-run] Would remove" if dry_run else "Removed" + try: + project_root = Path(project_path).resolve(strict=True) + changed = _unconfigure_project_bridge(project_root, dry_run) + if changed is not None: + lines.append(f"{prefix} edit bridge settings from {changed}") + if remove_credentials: + _, _, *directories = local_bridge_layout( + project_root, bridge_config_root + ) + for directory in directories: + if directory.is_dir(): + if not dry_run: + shutil.rmtree(directory) + lines.append(f"{prefix} {directory}") + except (OSError, ValueError, json.JSONDecodeError) as exc: + return False, f"Edit bridge cleanup failed: {exc}" - return True, f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}" + return True, "\n".join(lines) if __name__ == "__main__": @@ -285,10 +351,15 @@ def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: parser.add_argument("--remote-url") parser.add_argument("--remote-ca-file") parser.add_argument("--remote-origin") + parser.add_argument("--remove-credentials", action="store_true") args = parser.parse_args() if args.uninstall: - ok, msg = uninstall_claude_hooks(dry_run=args.dry_run) + ok, msg = uninstall_claude_hooks( + dry_run=args.dry_run, + project_path=args.project_path, + remove_credentials=args.remove_credentials, + ) else: ok, msg = install_claude_hooks( dry_run=args.dry_run, diff --git a/daem0nmcp/claude_hooks/post_edit.py b/daem0nmcp/claude_hooks/post_edit.py index b3b3320..3d15628 100644 --- a/daem0nmcp/claude_hooks/post_edit.py +++ b/daem0nmcp/claude_hooks/post_edit.py @@ -7,9 +7,11 @@ from collections.abc import Mapping from typing import Any -from ..edit_host import EditHostConfig, EditHostStateStore, native_edit_capture_body from ._client import read_hook_event -from .native_edit import configured_native_edit_tools, native_edit_relative_paths + +# Pairing writes this into the project's Claude settings; without it there is +# no edit bridge, so the heavy bridge modules are never imported. +_CREDENTIAL_ENV = "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" def _ok(response: object) -> bool: @@ -20,6 +22,16 @@ def _ok(response: object) -> bool: def handle_post_edit(event: Mapping[str, Any], project_path: str) -> bool: """Stage only a bounded host-generated candidate for a consumed request.""" try: + from ..edit_host import ( + EditHostConfig, + EditHostStateStore, + native_edit_capture_body, + ) + from .native_edit import ( + configured_native_edit_tools, + native_edit_relative_paths, + ) + native_session_id = event["session_id"] native_request_id = event["tool_use_id"] tool_name = event["tool_name"] @@ -69,7 +81,7 @@ def handle_post_edit(event: Mapping[str, Any], project_path: str) -> bool: def main() -> None: event = read_hook_event() path = event.get("cwd") - if isinstance(path, str) and path: + if isinstance(path, str) and path and os.environ.get(_CREDENTIAL_ENV): handle_post_edit(event, path) sys.exit(0) diff --git a/daem0nmcp/claude_hooks/pre_edit.py b/daem0nmcp/claude_hooks/pre_edit.py index a446d6c..fc34874 100644 --- a/daem0nmcp/claude_hooks/pre_edit.py +++ b/daem0nmcp/claude_hooks/pre_edit.py @@ -1,161 +1,67 @@ -"""Claude Code native-edit bridge adapter.""" +"""Claude Code PreToolUse hook for edits: remind, never block. + +Stdlib only (plus ``read_hook_event``) so it stays fast on every edit. Outside +a Daem0n project it is silent; inside one it adds a one-line reminder to the +model's context and always exits 0. +""" from __future__ import annotations +import json import os import sys -from collections.abc import Mapping -from datetime import datetime -from typing import Any - -from ..edit_host import ( - CreatedHostSession, - EditHostConfig, - EditHostStateStore, - PendingEditBinding, -) -from ._client import block, read_hook_event -from .native_edit import configured_native_edit_tools, native_edit_request - - -class PreEditResult: - def __init__(self, allowed: bool, message: str) -> None: - self.allowed, self.message = allowed, message - - -async def async_main(project_path: str, file_path: str) -> PreEditResult: - """Compatibility entry point; standalone calls cannot forge host sessions.""" - del project_path, file_path - return PreEditResult(False, "EDIT_BRIDGE_UNAVAILABLE") - +from pathlib import Path -def _expires(value: object) -> datetime: - if not isinstance(value, str): - raise ValueError - return datetime.fromisoformat(value.removesuffix("Z") + "+00:00") +from ._client import read_hook_event -def _data(response: object) -> Mapping[str, Any]: - body = getattr(response, "body", None) - if ( - not isinstance(body, Mapping) - or body.get("ok") is not True - or not isinstance(body.get("data"), Mapping) - ): - raise RuntimeError - return body["data"] - - -def handle_pre_edit(event: Mapping[str, Any], project_path: str) -> PreEditResult: +def _relative_path(project: Path, tool_input: object) -> str | None: + if not isinstance(tool_input, dict): + return None + raw = tool_input.get("file_path") or tool_input.get("notebook_path") + if not isinstance(raw, str) or not raw: + return None try: - native_session_id, native_request_id, tool_name, tool_input = ( - event["session_id"], - event["tool_use_id"], - event["tool_name"], - event["tool_input"], - ) - if ( - not isinstance(native_session_id, str) - or not isinstance(native_request_id, str) - or not isinstance(tool_name, str) - or not isinstance(tool_input, Mapping) - ): - raise ValueError - config = EditHostConfig.from_environment(os.environ) - workspace_id = config.workspace_id(project_path) - store = EditHostStateStore(config) - client = config.build_client() - edit = native_edit_request( - project_path=project_path, - tool_name=tool_name, - tool_input=tool_input, - configured_tools=configured_native_edit_tools(os.environ), - ) - - def create() -> CreatedHostSession: - data = _data(client.call("/v1/sessions", {"workspace_id": workspace_id})) - return CreatedHostSession( - str(data["host_session_id"]), _expires(data["expires_at"]) - ) - - host = store.load_or_create_session( - workspace_id=workspace_id, - native_session_id=native_session_id, - credential_id=config.identity.credential_id, - create=create, - ) - pending = store.get_pending( - workspace_id=workspace_id, - native_session_id=native_session_id, - credential_id=config.identity.credential_id, - edit_hash=edit.edit_hash, - ) - wire_edit = { - "tool_name": edit.tool_name, - "arguments": dict(edit.arguments), - "preimages": [item.canonical() for item in edit.preimages], + return (project / raw).resolve().relative_to(project.resolve()).as_posix() + except (OSError, ValueError): + return None + + +def reminder(event: dict) -> str | None: + """Return the reminder JSON for this event, or None to stay silent.""" + root = event.get("cwd") or os.environ.get("CLAUDE_PROJECT_DIR") + if not isinstance(root, str) or not root: + return None + project = Path(root) + if not (project / ".daem0nmcp").is_dir(): + return None + path = _relative_path(project, event.get("tool_input")) + recall = ( + f'memory_recall_file(relative_file_path="{path}")' + if path + else "memory_recall_file" + ) + text = ( + f"Daem0n: before this edit, call {recall} for past decisions and " + "warnings, and memory_preflight for the exact change." + ) + return json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": text, + } } - if pending is not None: - data = _data( - client.call( - "/v1/receipts/consume", - { - "workspace_id": workspace_id, - "host_session_id": host.host_session_id, - "edit_request_id": pending.edit_request_id, - "edit": wire_edit, - }, - ) - ) - if data.get("allowed") is not True: - raise RuntimeError - store.mark_consumed( - workspace_id=workspace_id, - native_session_id=native_session_id, - native_request_id=native_request_id, - credential_id=config.identity.credential_id, - edit_request_id=pending.edit_request_id, - ) - return PreEditResult(True, "") - data = _data( - client.call( - "/v1/edits", - { - "workspace_id": workspace_id, - "host_session_id": host.host_session_id, - "edit": wire_edit, - }, - ) - ) - store.save_pending( - native_session_id=native_session_id, - native_request_id=native_request_id, - credential_id=config.identity.credential_id, - pending=PendingEditBinding( - workspace_id, - host.host_session_id, - str(data["edit_request_id"]), - edit.edit_hash, - _expires(data["expires_at"]), - ), - ) - remedy = data.get("remedy") - return PreEditResult( - False, - str(remedy) if isinstance(remedy, Mapping) else "EDIT_BRIDGE_UNAVAILABLE", - ) - except Exception: - return PreEditResult(False, "EDIT_BRIDGE_UNAVAILABLE") + ) def main() -> None: - event = read_hook_event() - path = event.get("cwd") - if not isinstance(path, str) or not path: - block("EDIT_BRIDGE_UNAVAILABLE") - result = handle_pre_edit(event, path) - if not result.allowed: - block(result.message) + try: + output = reminder(read_hook_event()) + except Exception: + output = None + if output: + print(output) sys.exit(0) diff --git a/daem0nmcp/claude_hooks/stop.py b/daem0nmcp/claude_hooks/stop.py index 59766f3..c40d93e 100644 --- a/daem0nmcp/claude_hooks/stop.py +++ b/daem0nmcp/claude_hooks/stop.py @@ -1,8 +1,9 @@ -"""Claude Code Stop hook for fail-closed v7 memory reminders. +"""Claude Code Stop hook: remind about v7 memory calls, never block. -The hook analyzes the transcript but never writes memory directly. It emits -scoped, replay-safe ``memory_store`` and ``memory_record_outcome`` suggestions -that the authenticated MCP host can execute. +The hook reads the stdin event (``cwd``, ``transcript_path``, ``session_id``), +analyzes the transcript, and never writes memory. Its replay-safe +``memory_store`` and ``memory_record_outcome`` suggestions are shown to the +user as a ``systemMessage``; it never emits ``decision`` and always exits 0. """ import contextlib @@ -14,7 +15,7 @@ from pathlib import Path from ..workspace import WorkspaceRegistry -from ._client import get_project_path, run_async, succeed +from ._client import read_hook_event, run_async # ─── transcript analysis ─────────────────────────────────────────── @@ -86,14 +87,13 @@ def _state_dir() -> Path: return d -def _state_file() -> Path: - session_id = os.environ.get("CLAUDE_SESSION_ID", "default") +def _state_file(session_id: str) -> Path: safe = re.sub(r"[^\w\-]", "_", session_id) return _state_dir() / f"stop_{safe}.json" -def _load_state() -> dict: - f = _state_file() +def _load_state(session_id: str) -> dict: + f = _state_file(session_id) if f.exists(): try: return json.loads(f.read_text(encoding="utf-8")) @@ -102,17 +102,17 @@ def _load_state() -> dict: return {"reminder_count": 0, "last_reminder_turn": -1} -def _save_state(state: dict) -> None: +def _save_state(session_id: str, state: dict) -> None: with contextlib.suppress(OSError): - _state_file().write_text(json.dumps(state), encoding="utf-8") + _state_file(session_id).write_text(json.dumps(state), encoding="utf-8") # ─── transcript reading ─────────────────────────────────────────── -def _read_transcript() -> list[dict]: - path = os.environ.get("CLAUDE_TRANSCRIPT_PATH", "") - if not path or not Path(path).exists(): +def _read_transcript(path: object) -> list[dict]: + """Read Claude Code JSONL records, unwrapping the nested ``message``.""" + if not isinstance(path, str) or not path or not Path(path).is_file(): return [] messages = [] try: @@ -121,9 +121,15 @@ def _read_transcript() -> list[dict]: line = line.strip() if line: try: - messages.append(json.loads(line)) + record = json.loads(line) except json.JSONDecodeError: continue + if isinstance(record, dict) and isinstance( + record.get("message"), dict + ): + record = record["message"] + if isinstance(record, dict): + messages.append(record) except OSError: pass return messages @@ -359,20 +365,28 @@ async def analyse_and_remember( def main() -> None: - project_path = get_project_path() - if project_path is None: + event = read_hook_event() + project_path = event.get("cwd") or os.environ.get("CLAUDE_PROJECT_DIR") + if not isinstance(project_path, str) or not project_path: + sys.exit(0) + if not (Path(project_path) / ".daem0nmcp").is_dir(): sys.exit(0) - messages = _read_transcript() + messages = _read_transcript(event.get("transcript_path")) if not messages: sys.exit(0) - state = _load_state() + session_id = event.get("session_id") + if not isinstance(session_id, str) or not session_id: + session_id = "default" + state = _load_state(session_id) result = run_async(analyse_and_remember(project_path, messages, state)) - _save_state(state) + _save_state(session_id, state) if result.message: - succeed(result.message) + # systemMessage is shown to the user; the hook never returns a + # "decision", so it cannot keep the agent running. + print(json.dumps({"systemMessage": result.message})) sys.exit(0) diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..05b3c6a 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -16,7 +16,7 @@ python -m daem0nmcp.cli install-hooks [--force] python -m daem0nmcp.cli uninstall-hooks python -m daem0nmcp.cli install-claude-hooks [--dry-run] - python -m daem0nmcp.cli uninstall-claude-hooks [--dry-run] + python -m daem0nmcp.cli [--project-path P] uninstall-claude-hooks [--dry-run] [--remove-credentials] python -m daem0nmcp.cli install-opencode [--dry-run] [--force] python -m daem0nmcp.cli watch [--debounce SECONDS] [--no-system] [--no-log] [--no-poll] python -m daem0nmcp.cli index [--path PATH] [--patterns *.py *.ts ...] @@ -470,6 +470,11 @@ def main(): uninstall_claude_parser.add_argument( "--dry-run", action="store_true", help="Show what would change" ) + uninstall_claude_parser.add_argument( + "--remove-credentials", + action="store_true", + help="Also delete the project's local edit-bridge credential (with --project-path)", + ) # install-opencode command install_oc_parser = subparsers.add_parser( @@ -929,7 +934,9 @@ def main(): from .claude_hooks.install import uninstall_claude_hooks success, message = uninstall_claude_hooks( - dry_run=getattr(args, "dry_run", False) + dry_run=getattr(args, "dry_run", False), + project_path=args.project_path, + remove_credentials=args.remove_credentials, ) if args.json: print(json.dumps({"success": success, "message": message})) diff --git a/daem0nmcp/edit_host.py b/daem0nmcp/edit_host.py index 2bb3d08..1f834e6 100644 --- a/daem0nmcp/edit_host.py +++ b/daem0nmcp/edit_host.py @@ -590,14 +590,11 @@ def provision_client_bridge_installation( ) -def provision_local_bridge_installation( - project_root: str | Path, - *, - config_root: Path | None = None, -) -> LocalBridgeInstallation: - """Provision or validate the local host credential outside the workspace.""" +def local_bridge_layout( + root: Path, config_root: Path | None +) -> tuple[Path, str, Path, Path]: + """Return storage path, principal, credential dir and socket dir for *root*.""" - root = Path(project_root).resolve(strict=True) settings = Settings(project_root=str(root)) storage_path = Path(settings.get_storage_path()).resolve(strict=False) principal_id = local_authority_principal(storage_path) @@ -608,12 +605,30 @@ def provision_local_bridge_installation( else Path.home() / ".daem0nmcp" / "edit-bridges" ) reject_linked_ancestry(base) - authority_directory = ensure_owner_only_directory(base.absolute() / authority_id) - credential_path = authority_directory / "credential.json" # AF_UNIX socket paths are limited to 104-108 bytes, and the authority # directory name alone is 64 characters, so sockets live in a short # sibling directory instead. - runtime_directory = base.absolute() / "run" / authority_id[:16] + return ( + storage_path, + principal_id, + base.absolute() / authority_id, + base.absolute() / "run" / authority_id[:16], + ) + + +def provision_local_bridge_installation( + project_root: str | Path, + *, + config_root: Path | None = None, +) -> LocalBridgeInstallation: + """Provision or validate the local host credential outside the workspace.""" + + root = Path(project_root).resolve(strict=True) + storage_path, principal_id, authority, runtime_directory = local_bridge_layout( + root, config_root + ) + authority_directory = ensure_owner_only_directory(authority) + credential_path = authority_directory / "credential.json" created = False if credential_path.exists(): _, identity = load_bridge_credential(credential_path) diff --git a/hooks/daem0n_post_edit_hook.py b/hooks/daem0n_post_edit_hook.py index 0a4397f..fb431b5 100644 --- a/hooks/daem0n_post_edit_hook.py +++ b/hooks/daem0n_post_edit_hook.py @@ -1,141 +1,11 @@ #!/usr/bin/env python3 -"""Read-only Claude PostToolUse guidance for significant v7 changes.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os import sys -from pathlib import Path - -SIGNIFICANT_EXTENSIONS = frozenset( - { - ".py", - ".ts", - ".js", - ".go", - ".rs", - ".java", - ".yaml", - ".yml", - ".json", - ".toml", - ".sql", - } -) -SIGNIFICANT_TERMS = ( - "async def ", - "class ", - "auth", - "config", - "credential", - "endpoint", - "migration", - "schema", - "secret", - "token", +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, ) - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _tool_input() -> dict[str, object]: - try: - value = json.loads(os.environ.get("TOOL_INPUT", "{}")) - except (json.JSONDecodeError, TypeError): - return {} - return value if isinstance(value, dict) else {} - - -def _relative_path(root: Path, value: str) -> str | None: - try: - return Path(value).resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None - - -def _is_significant(value: str, changed: str) -> bool: - if Path(value).suffix.casefold() not in SIGNIFICANT_EXTENSIONS: - return False - lowered = changed.casefold() - return len(changed) > 500 or any(term in lowered for term in SIGNIFICANT_TERMS) - - -def _suggestion(root: Path, value: str, changed: str) -> str: - key = os.path.normcase(str(root)) - workspace_id = "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - relative_path = _relative_path(root, value) - path_label = relative_path or Path(value).name or "changed file" - digest = hashlib.sha256( - f"{workspace_id}\0{path_label}\0{changed}".encode("utf-8") - ).hexdigest()[:24] - target_arguments: dict[str, object] = { - "record_type": "decision", - "content": f"Summarize the significant change to {path_label}", - "rationale": "Preserve the reason and constraints for future sessions", - "idempotency_key": f"hook-postedit-{digest}", - } - if relative_path is not None: - target_arguments["relative_file_path"] = relative_path - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - store_fields = [ - f'workspace_id="{workspace_id}"', - 'record_type="decision"', - f'content={json.dumps(target_arguments["content"])}', - f'rationale={json.dumps(target_arguments["rationale"])}', - ] - if relative_path is not None: - store_fields.append(f"relative_file_path={json.dumps(relative_path)}") - store_fields.extend( - ( - f'idempotency_key="{target_arguments["idempotency_key"]}"', - 'preflight_token=""', - ) - ) - return ( - "[Daem0n suggests] This hook did not write memory. First call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - f'description="Record the significant change to {path_label}"). Then ' - "call mcp__daem0nmcp__memory_store(" + ", ".join(store_fields) + ")." - ) - - -def main() -> None: - root = _configured_root() - if root is None: - return - data = _tool_input() - value = data.get("file_path") - if not isinstance(value, str) or not value: - return - changed = " ".join( - item - for item in ( - data.get("old_string"), - data.get("new_string"), - data.get("content"), - ) - if isinstance(item, str) - ) - if _is_significant(value, changed): - print(_suggestion(root, value, changed)) - - -if __name__ == "__main__": - try: - main() - except Exception: - print( - "[Daem0n v7 warning] No memory was written. Call memory_preflight " - "before any memory_store retry, and use system_health for diagnostics." - ) - sys.exit(0) +sys.exit(0) diff --git a/hooks/daem0n_pre_edit_hook.py b/hooks/daem0n_pre_edit_hook.py index 5dc0a56..fb431b5 100644 --- a/hooks/daem0n_pre_edit_hook.py +++ b/hooks/daem0n_pre_edit_hook.py @@ -1,99 +1,11 @@ #!/usr/bin/env python3 -"""Fail-closed Claude PreToolUse guidance for v7-scoped edits.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os import sys -from pathlib import Path - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _tool_path() -> str | None: - try: - data = json.loads(os.environ.get("TOOL_INPUT", "{}")) - except (json.JSONDecodeError, TypeError): - return None - if not isinstance(data, dict): - return None - value = data.get("file_path") or data.get("notebook_path") - return value if isinstance(value, str) and value else None - - -def _relative_path(root: Path, value: str) -> str | None: - try: - return Path(value).resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None - - -def _workspace_id(root: Path) -> str: - key = os.path.normcase(str(root)) - return "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - - -def _guidance(root: Path, value: str) -> str: - workspace_id = _workspace_id(root) - relative_path = _relative_path(root, value) - path_label = relative_path or "" - digest = hashlib.sha256( - f"{workspace_id}\0{path_label}".encode("utf-8") - ).hexdigest()[:24] - target_arguments = { - "record_type": "decision", - "content": f"Describe the planned change to {path_label}", - "idempotency_key": f"hook-preedit-{digest}", - } - if relative_path is not None: - target_arguments["relative_file_path"] = relative_path - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - return ( - "[Daem0n blocks] This standalone hook cannot access the authenticated " - "v7 invocation, so it fails closed. Ask the MCP host to call " - f'mcp__daem0nmcp__memory_recall(workspace_id="{workspace_id}", ' - f'query="planned edit to {path_label}", limit=10). If the plan is ' - "durable, call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - f'description="Review the planned edit to {path_label}"). Review the ' - "returned warnings and use its preflight_token only with that exact " - "memory_store request." - ) - - -def main() -> int: - root = _configured_root() - if root is None: - return 0 - value = _tool_path() - if value is None: - print( - "[Daem0n blocks] Unable to identify the edit target; this hook " - "fails closed. Call memory_recall and memory_preflight through the " - "authenticated MCP host.", - file=sys.stderr, - ) - return 2 - try: - message = _guidance(root, value) - except Exception: - message = ( - "[Daem0n blocks] Unable to construct scoped v7 guidance; this " - "hook fails closed. Call session_brief, memory_recall, and " - "memory_preflight through the authenticated MCP host." - ) - print(message, file=sys.stderr) - return 2 - -if __name__ == "__main__": - sys.exit(main()) +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, +) +sys.exit(0) diff --git a/hooks/daem0n_prompt_hook.py b/hooks/daem0n_prompt_hook.py index 9b1f2ea..fb431b5 100755 --- a/hooks/daem0n_prompt_hook.py +++ b/hooks/daem0n_prompt_hook.py @@ -1,49 +1,11 @@ #!/usr/bin/env python3 -"""Read-only Claude UserPromptSubmit guidance for the v7 ritual.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import os import sys -from pathlib import Path - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _workspace_id(root: Path) -> str: - key = os.path.normcase(str(root)) - return "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - - -def main() -> None: - """Tell the authenticated host which bounded read calls to make.""" - root = _configured_root() - if root is None: - return - workspace_id = _workspace_id(root) - print( - "[Daem0n v7 reminder] At session start call " - f'mcp__daem0nmcp__session_brief(workspace_id="{workspace_id}"). ' - "When history is relevant, call " - f'mcp__daem0nmcp__memory_recall(workspace_id="{workspace_id}", ' - 'query="", limit=10). Use only the opaque workspace_id ' - "issued for this workspace." - ) - -if __name__ == "__main__": - try: - main() - except Exception: - print( - "[Daem0n v7 reminder] Call session_brief before protected work; " - "use system_health if the scoped briefing cannot be obtained." - ) - sys.exit(0) +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, +) +sys.exit(0) diff --git a/hooks/daem0n_stop_hook.py b/hooks/daem0n_stop_hook.py index db8f91c..fb431b5 100755 --- a/hooks/daem0n_stop_hook.py +++ b/hooks/daem0n_stop_hook.py @@ -1,149 +1,11 @@ #!/usr/bin/env python3 -"""Stateless, read-only Claude Stop guidance for the v7 outcome ritual.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os -import re import sys -from pathlib import Path - -_MAX_TRANSCRIPT_BYTES = 1_000_000 -_MAX_MESSAGES = 50 -_COMPLETION = re.compile( - r"(?i)\b(?:implementation|task|work|feature)\s+(?:is\s+)?" - r"(?:complete|done|finished)\b|\ball tasks?\s+(?:are\s+)?done\b" -) -_DECISION = re.compile( - r"(?i)\bi (?:will|chose|decided to)\s+(.{20,240}?)(?:\.|$)" -) -_OUTCOME_TOOL_NAMES = frozenset( - { - "memory_record_outcome", - "daem0nmcp_memory_record_outcome", - "mcp__daem0nmcp__memory_record_outcome", - } +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, ) - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _messages() -> list[dict[str, object]]: - raw_path = os.environ.get("CLAUDE_TRANSCRIPT_PATH", "") - if not raw_path: - return [] - path = Path(raw_path) - try: - if not path.is_file() or path.stat().st_size > _MAX_TRANSCRIPT_BYTES: - return [] - lines = path.read_text(encoding="utf-8").splitlines()[-_MAX_MESSAGES:] - except OSError: - return [] - messages: list[dict[str, object]] = [] - for line in lines: - try: - value = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(value, dict): - messages.append(value) - return messages - - -def _recent_content(messages: list[dict[str, object]]) -> tuple[str, set[str]]: - text_parts: list[str] = [] - tools: set[str] = set() - for message in messages: - if message.get("role") != "assistant": - continue - content = message.get("content") - if isinstance(content, str): - text_parts.append(content) - elif isinstance(content, list): - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") == "text" and isinstance(part.get("text"), str): - text_parts.append(str(part["text"])) - if part.get("type") == "tool_use" and isinstance(part.get("name"), str): - tools.add(str(part["name"]).casefold()) - return " ".join(text_parts)[-20_000:], tools - - -def _reason(root: Path, content: str) -> str: - key = os.path.normcase(str(root)) - workspace_id = "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - match = _DECISION.search(content) - decision = ( - re.sub(r"\s+", " ", match.group(1)).strip() - if match is not None - else "Summarize the durable decision from the completed task" - ) - digest = hashlib.sha256( - f"{workspace_id}\0{decision}".encode("utf-8") - ).hexdigest()[:24] - store_key = f"hook-stop-{digest}" - outcome_key = f"hook-outcome-{digest}" - target_arguments = { - "record_type": "decision", - "content": decision, - "rationale": "Preserve the verified decision from the completed task", - "idempotency_key": store_key, - } - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - return ( - "[Daem0n blocks] Completion was detected and this hook did not mutate " - "memory. Review and call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - 'description="Record the completed task decision"); then call ' - f'mcp__daem0nmcp__memory_store(workspace_id="{workspace_id}", ' - f'record_type="decision", content={json.dumps(decision)}, ' - 'rationale="Preserve the verified decision from the completed task", ' - f'idempotency_key="{store_key}", ' - 'preflight_token=""). Once verified, call ' - f'mcp__daem0nmcp__memory_record_outcome(workspace_id="{workspace_id}", ' - 'record_id="", outcome_text="", worked=true, ' - f'idempotency_key="{outcome_key}").' - ) - - -def main() -> None: - root = _configured_root() - if root is None: - return - content, tools = _recent_content(_messages()) - if not content or not _COMPLETION.search(content): - return - if tools.intersection(_OUTCOME_TOOL_NAMES): - return - print(json.dumps({"decision": "block", "reason": _reason(root, content)})) - - -if __name__ == "__main__": - try: - main() - except Exception: - print( - json.dumps( - { - "decision": "block", - "reason": ( - "[Daem0n blocks] Unable to verify v7 outcome state. " - "Call session_brief and system_health, then use " - "memory_preflight, memory_store, and " - "memory_record_outcome through the authenticated host." - ), - } - ) - ) - sys.exit(0) +sys.exit(0) diff --git a/hooks/settings.json.example b/hooks/settings.json.example deleted file mode 100644 index 7cc7d38..0000000 --- a/hooks/settings.json.example +++ /dev/null @@ -1,70 +0,0 @@ -{ - "hooks": { - "SessionStart": [ - { - "matcher": "startup|resume", - "hooks": [ - { - "type": "command", - "command": "echo '[Daem0n v7] Before responding, call mcp__daem0nmcp__session_brief(workspace_id=\"\").'" - } - ] - } - ], - "UserPromptSubmit": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_prompt_hook.py\"" - } - ] - } - ], - "PreToolUse": [ - { - "matcher": "Edit|Write|NotebookEdit", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_pre_edit_hook.py\"" - } - ] - } - ], - "PostToolUse": [ - { - "matcher": "Edit|Write", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_post_edit_hook.py\"" - } - ] - } - ], - "Stop": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - } - ] - } - ], - "SubagentStop": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - } - ] - } - ] - } -} diff --git a/tests/api_v7/test_docs_hooks.py b/tests/api_v7/test_docs_hooks.py index 0fdc00e..dff4413 100644 --- a/tests/api_v7/test_docs_hooks.py +++ b/tests/api_v7/test_docs_hooks.py @@ -35,7 +35,6 @@ ROOT / "hooks" / "daem0n_pre_edit_hook.py", ROOT / "hooks" / "daem0n_post_edit_hook.py", ROOT / "hooks" / "daem0n_stop_hook.py", - ROOT / "hooks" / "settings.json.example", ) ROOT_HOOKS = tuple(path for path in CUTOVER_FILES if path.parent == ROOT / "hooks") @@ -100,7 +99,7 @@ def _run_root_hook( name: str, *, workspace_root: Path, - extra_environment: dict[str, str] | None = None, + event: dict[str, object], ) -> subprocess.CompletedProcess[str]: environment = os.environ.copy() environment.update( @@ -111,12 +110,11 @@ def _run_root_hook( "PYTHONDONTWRITEBYTECODE": "1", } ) - if extra_environment: - environment.update(extra_environment) return subprocess.run( [sys.executable, str(ROOT / "hooks" / name)], - cwd=ROOT, + cwd=workspace_root, env=environment, + input=json.dumps(event), capture_output=True, text=True, timeout=5, @@ -221,42 +219,15 @@ def test_hooks_do_not_open_the_mutable_v6_manager_path(self) -> None: self.assertNotIn("memory.remember", hook_sources) self.assertNotIn("INSERT INTO session_state", hook_sources) - def test_root_hooks_are_read_only_v7_host_guidance(self) -> None: - hook_sources = "\n".join(_read(path) for path in ROOT_HOOKS) - - for forbidden in ( - "subprocess", - "daem0nmcp.cli", - "os.system", - "os.popen", - "urllib.request", - "sqlite3", - ".write_text(", - ".write_bytes(", - ".mkdir(", - ".touch(", - ".unlink(", - "open(", - ): - with self.subTest(forbidden=forbidden): - self.assertNotIn(forbidden, hook_sources) - - expected_calls = { - "daem0n_prompt_hook.py": ("session_brief", "memory_recall"), - "daem0n_pre_edit_hook.py": ("memory_recall", "memory_preflight"), - "daem0n_post_edit_hook.py": ("memory_preflight", "memory_store"), - "daem0n_stop_hook.py": ( - "memory_preflight", - "memory_store", - "memory_record_outcome", - ), - "settings.json.example": ("session_brief",), - } + def test_root_hooks_are_stdlib_only_deprecation_stubs(self) -> None: for path in ROOT_HOOKS: text = _read(path) - for tool_name in expected_calls[path.name]: - with self.subTest(path=path.name, tool_name=tool_name): - self.assertIn(tool_name, text) + with self.subTest(path=path.name): + imports = re.findall(r"^\s*(?:import|from)\s+(\S+)", text, re.M) + self.assertEqual(["sys"], imports) + self.assertIn("install-claude-hooks", text) + self.assertIn("sys.exit(0)", text) + self.assertNotIn("exit(2)", text) class HookNameTests(unittest.TestCase): @@ -321,103 +292,56 @@ async def test_stop_suggests_replay_safe_v7_calls_without_writing(self) -> None: self.assertIn("preflight_token", result.message) self.assertFalse((project / ".daem0nmcp" / "storage").exists()) - async def test_pre_edit_fails_closed_without_native_bridge(self) -> None: - from daem0nmcp.claude_hooks.pre_edit import async_main + def test_pre_edit_reminds_without_blocking(self) -> None: + from daem0nmcp.claude_hooks.pre_edit import reminder with tempfile.TemporaryDirectory() as tmp_dir: project = Path(tmp_dir) + event = { + "cwd": str(project), + "tool_name": "Edit", + "tool_input": {"file_path": str(project / "server.py")}, + } + self.assertIsNone(reminder(event)) (project / ".daem0nmcp").mkdir() - result = await async_main(str(project), str(project / "server.py")) + output = json.loads(reminder(event) or "{}")["hookSpecificOutput"] - self.assertFalse(result.allowed) - self.assertEqual("EDIT_BRIDGE_UNAVAILABLE", result.message) + self.assertNotIn("permissionDecision", output) + self.assertIn("memory_preflight", output["additionalContext"]) + self.assertIn( + 'memory_recall_file(relative_file_path="server.py")', + output["additionalContext"], + ) self.assertFalse((project / ".daem0nmcp" / "storage").exists()) class RootHookProcessTests(unittest.TestCase): - def test_pre_edit_process_denies_without_mutating_workspace(self) -> None: - with tempfile.TemporaryDirectory() as tmp_dir: - workspace_root = Path(tmp_dir) - (workspace_root / ".daem0nmcp").mkdir() - before = _tree_snapshot(workspace_root) - - result = _run_root_hook( - "daem0n_pre_edit_hook.py", - workspace_root=workspace_root, - extra_environment={ - "TOOL_INPUT": json.dumps( - {"file_path": str(workspace_root / "service.py")} - ) - }, - ) - - self.assertEqual(2, result.returncode) - self.assertEqual("", result.stdout) - self.assertIn("fails closed", result.stderr) - self.assertIn("memory_recall", result.stderr) - self.assertIn("memory_preflight", result.stderr) - self.assertRegex(result.stderr, r"workspace_id=[\"']ws_[a-f0-9]{24}") - self.assertIn("target_tool", result.stderr) - self.assertIn("target_arguments", result.stderr) - self.assertEqual(before, _tree_snapshot(workspace_root)) - - def test_advisory_processes_emit_exact_v7_calls_without_writing(self) -> None: + def test_legacy_stubs_exit_zero_on_a_pre_tool_use_payload(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: workspace_root = Path(tmp_dir) (workspace_root / ".daem0nmcp").mkdir() - transcript = workspace_root / "transcript.jsonl" - transcript.write_text( - json.dumps( - { - "role": "assistant", - "content": ( - "I will use signed cookies because they avoid shared " - "server state. Implementation is complete." - ), - } - ) - + "\n", - encoding="utf-8", - ) before = _tree_snapshot(workspace_root) - tool_input = json.dumps( - { + event = { + "session_id": "s-1", + "cwd": str(workspace_root), + "hook_event_name": "PreToolUse", + "tool_name": "Edit", + "tool_input": { "file_path": str(workspace_root / "service.py"), - "new_string": "async def authenticate(token): pass", - } - ) - - prompt = _run_root_hook( - "daem0n_prompt_hook.py", - workspace_root=workspace_root, - ) - post_edit = _run_root_hook( - "daem0n_post_edit_hook.py", - workspace_root=workspace_root, - extra_environment={"TOOL_INPUT": tool_input}, - ) - stop = _run_root_hook( - "daem0n_stop_hook.py", - workspace_root=workspace_root, - extra_environment={ - "CLAUDE_TRANSCRIPT_PATH": str(transcript), + "old_string": "a", + "new_string": "b", }, - ) + } - self.assertEqual(0, prompt.returncode) - self.assertIn("session_brief", prompt.stdout) - self.assertIn("memory_recall", prompt.stdout) - self.assertEqual(0, post_edit.returncode) - self.assertIn("memory_preflight", post_edit.stdout) - self.assertIn("memory_store", post_edit.stdout) - self.assertIn("idempotency_key", post_edit.stdout) - self.assertEqual(0, stop.returncode) - stop_payload = json.loads(stop.stdout) - self.assertEqual("block", stop_payload["decision"]) - self.assertIn("memory_preflight", stop_payload["reason"]) - self.assertIn("memory_store", stop_payload["reason"]) - self.assertIn("memory_record_outcome", stop_payload["reason"]) - self.assertIn("idempotency_key", stop_payload["reason"]) + for path in ROOT_HOOKS: + with self.subTest(path=path.name): + result = _run_root_hook( + path.name, workspace_root=workspace_root, event=event + ) + self.assertEqual(0, result.returncode) + self.assertEqual("", result.stdout) + self.assertIn("deprecated", result.stderr) + self.assertIn("install-claude-hooks", result.stderr) self.assertEqual(before, _tree_snapshot(workspace_root)) diff --git a/tests/api_v7/test_process_edit_capture.py b/tests/api_v7/test_process_edit_capture.py index 6118ab9..b226d46 100644 --- a/tests/api_v7/test_process_edit_capture.py +++ b/tests/api_v7/test_process_edit_capture.py @@ -12,7 +12,6 @@ from daem0nmcp.claude_hooks.post_edit_preflight import ( handle_edit_preflight_response, ) -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.database import DatabaseManager from daem0nmcp.edit_host import ( EditHostConfig, @@ -21,6 +20,7 @@ ) from daem0nmcp.workspace import WorkspaceRegistry from tests.api_v7.process_client import call, process_client, succeed +from tests.native_edit_host import drive_native_edit @pytest.fixture @@ -68,8 +68,8 @@ async def test_actual_stdio_mcp_shares_bridge_authority_and_promotes_capture( ) as session: await succeed(session, "session_brief", scope) with patch.dict(os.environ, environment, clear=False): - denied = handle_pre_edit(first_event, str(workspace.root)) - assert not denied.allowed + denied = drive_native_edit(first_event, str(workspace.root)) + assert not denied normalized = native_edit_request( project_path=workspace.root, tool_name="Edit", @@ -112,8 +112,8 @@ async def test_actual_stdio_mcp_shares_bridge_authority_and_promotes_capture( ) retry_event = {**first_event, "tool_use_id": "tool-use-allowed-2"} with patch.dict(os.environ, environment, clear=False): - allowed = handle_pre_edit(retry_event, str(workspace.root)) - assert allowed.allowed + allowed = drive_native_edit(retry_event, str(workspace.root)) + assert allowed target.write_text("after", encoding="utf-8") with patch.dict(os.environ, environment, clear=False): assert handle_post_edit(retry_event, str(workspace.root)) diff --git a/tests/native_edit_host.py b/tests/native_edit_host.py new file mode 100644 index 0000000..e820e12 --- /dev/null +++ b/tests/native_edit_host.py @@ -0,0 +1,121 @@ +"""Test-only native host driver for the edit bridge. + +The Claude ``pre_edit`` hook no longer drives the bridge (it only reminds), but +the bridge itself stays until its planned removal. These steps are what a +native host does before an edit: open a host session, then either consume an +approved receipt or create a pending edit request. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from daem0nmcp.claude_hooks.native_edit import ( + configured_native_edit_tools, + native_edit_request, +) +from daem0nmcp.edit_host import ( + CreatedHostSession, + EditHostConfig, + EditHostStateStore, + PendingEditBinding, +) + + +def _expires(value: object) -> datetime: + assert isinstance(value, str) + return datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + + +def _data(response: object) -> Mapping[str, Any]: + body = getattr(response, "body", None) + assert isinstance(body, Mapping) and body.get("ok") is True, body + return body["data"] + + +def drive_native_edit(event: Mapping[str, Any], project_path: str) -> bool: + """Return True when an approved receipt was consumed for this exact edit.""" + native_session_id = event["session_id"] + native_request_id = event["tool_use_id"] + config = EditHostConfig.from_environment(os.environ) + workspace_id = config.workspace_id(project_path) + store = EditHostStateStore(config) + client = config.build_client() + edit = native_edit_request( + project_path=project_path, + tool_name=event["tool_name"], + tool_input=event["tool_input"], + configured_tools=configured_native_edit_tools(os.environ), + ) + + def create() -> CreatedHostSession: + data = _data(client.call("/v1/sessions", {"workspace_id": workspace_id})) + return CreatedHostSession( + str(data["host_session_id"]), _expires(data["expires_at"]) + ) + + credential_id = config.identity.credential_id + host = store.load_or_create_session( + workspace_id=workspace_id, + native_session_id=native_session_id, + credential_id=credential_id, + create=create, + ) + pending = store.get_pending( + workspace_id=workspace_id, + native_session_id=native_session_id, + credential_id=credential_id, + edit_hash=edit.edit_hash, + ) + wire_edit = { + "tool_name": edit.tool_name, + "arguments": dict(edit.arguments), + "preimages": [item.canonical() for item in edit.preimages], + } + if pending is not None: + data = _data( + client.call( + "/v1/receipts/consume", + { + "workspace_id": workspace_id, + "host_session_id": host.host_session_id, + "edit_request_id": pending.edit_request_id, + "edit": wire_edit, + }, + ) + ) + assert data.get("allowed") is True, data + store.mark_consumed( + workspace_id=workspace_id, + native_session_id=native_session_id, + native_request_id=native_request_id, + credential_id=credential_id, + edit_request_id=pending.edit_request_id, + ) + return True + data = _data( + client.call( + "/v1/edits", + { + "workspace_id": workspace_id, + "host_session_id": host.host_session_id, + "edit": wire_edit, + }, + ) + ) + store.save_pending( + native_session_id=native_session_id, + native_request_id=native_request_id, + credential_id=credential_id, + pending=PendingEditBinding( + workspace_id, + host.host_session_id, + str(data["edit_request_id"]), + edit.edit_hash, + _expires(data["expires_at"]), + ), + ) + return False diff --git a/tests/test_claude_hooks_client.py b/tests/test_claude_hooks_client.py index 356602b..40b7144 100644 --- a/tests/test_claude_hooks_client.py +++ b/tests/test_claude_hooks_client.py @@ -9,9 +9,7 @@ from daem0nmcp.claude_hooks._client import ( block, get_command_from_input, - get_file_path_from_input, get_project_path, - get_tool_input, read_hook_event, run_hook_safely, succeed, @@ -108,33 +106,17 @@ def test_rejects_oversized_or_non_object_stdin_event(self, monkeypatch): monkeypatch.setattr(sys, "stdin", TextIOWrapper(BytesIO(b"[]"))) assert read_hook_event() == {} - def test_file_path_from_edit(self, monkeypatch): - monkeypatch.setenv("TOOL_INPUT", json.dumps({"file_path": "/foo/bar.py"})) - assert get_file_path_from_input() == "/foo/bar.py" - - def test_notebook_path(self, monkeypatch): - monkeypatch.setenv("TOOL_INPUT", json.dumps({"notebook_path": "/foo.ipynb"})) - assert get_file_path_from_input() == "/foo.ipynb" - - def test_file_path_takes_precedence(self, monkeypatch): - monkeypatch.setenv( - "TOOL_INPUT", - json.dumps({"file_path": "/a.py", "notebook_path": "/b.ipynb"}), - ) - assert get_file_path_from_input() == "/a.py" - def test_command_from_bash(self, monkeypatch): monkeypatch.setenv("TOOL_INPUT", json.dumps({"command": "ls -la"})) assert get_command_from_input() == "ls -la" def test_invalid_json(self, monkeypatch): monkeypatch.setenv("TOOL_INPUT", "not json") - assert get_tool_input() == {} - assert get_file_path_from_input() is None + assert get_command_from_input() is None def test_missing_env(self, monkeypatch): monkeypatch.delenv("TOOL_INPUT", raising=False) - assert get_tool_input() == {} + assert get_command_from_input() is None class TestRunHookSafely: diff --git a/tests/test_claude_hooks_install.py b/tests/test_claude_hooks_install.py index 867e4f4..17a1676 100644 --- a/tests/test_claude_hooks_install.py +++ b/tests/test_claude_hooks_install.py @@ -295,3 +295,99 @@ def test_dry_run_no_write(self, fake_settings): # File should be unchanged assert fake_settings.read_text() == original + + def test_removes_legacy_manual_hook_entries(self, fake_settings): + fake_settings.write_text( + json.dumps( + { + "hooks": { + name: [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": f'python3 "$HOME/Daem0nMCP/hooks/{script}"', + } + ], + } + ] + for name, script in ( + ("UserPromptSubmit", "daem0n_prompt_hook.py"), + ("PreToolUse", "daem0n_pre_edit_hook.py"), + ("PostToolUse", "daem0n_post_edit_hook.py"), + ("Stop", "daem0n_stop_hook.py"), + ) + } + } + ) + ) + + ok, _msg = uninstall_claude_hooks() + + assert ok + assert json.loads(fake_settings.read_text())["hooks"] == {} + + def test_project_cleanup_strips_pairing_env_and_credentials( + self, fake_settings, tmp_path + ): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + ok, message = install_claude_hooks( + project_path=project, bridge_config_root=host_config + ) + assert ok, message + local_settings = project / ".claude" / "settings.local.json" + value = json.loads(local_settings.read_text(encoding="utf-8")) + value["env"]["USER_KEY"] = "kept" + value["permissions"] = {"allow": ["Bash(ls)"]} + local_settings.write_text(json.dumps(value), encoding="utf-8") + credential = Path(value["env"]["DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE"]) + runtime = Path(value["env"]["DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR"]) + runtime.mkdir(parents=True) + + ok, message = uninstall_claude_hooks( + dry_run=True, + project_path=project, + remove_credentials=True, + bridge_config_root=host_config, + ) + assert ok, message + assert credential.is_file() and runtime.is_dir() + assert "DAEM0NMCP_PROJECT_ROOT" in local_settings.read_text(encoding="utf-8") + + ok, message = uninstall_claude_hooks( + project_path=project, + remove_credentials=True, + bridge_config_root=host_config, + ) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == { + "env": {"USER_KEY": "kept"}, + "permissions": {"allow": ["Bash(ls)"]}, + } + assert not credential.parent.exists() + assert not runtime.exists() + assert runtime.parent.name == "run" + + def test_project_cleanup_without_credentials_keeps_them( + self, fake_settings, tmp_path + ): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + install_claude_hooks(project_path=project, bridge_config_root=host_config) + local_settings = project / ".claude" / "settings.local.json" + credential = Path( + json.loads(local_settings.read_text(encoding="utf-8"))["env"][ + "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" + ] + ) + + ok, message = uninstall_claude_hooks(project_path=project) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == {} + assert credential.is_file() diff --git a/tests/test_claude_hooks_post_edit.py b/tests/test_claude_hooks_post_edit.py index 8c56a63..693bab8 100644 --- a/tests/test_claude_hooks_post_edit.py +++ b/tests/test_claude_hooks_post_edit.py @@ -1,101 +1,62 @@ """Tests for the post_edit Claude Code hook.""" import json - -import pytest - -from daem0nmcp.claude_hooks.post_edit import main - - -def _set_env(monkeypatch, project_dir, tool_input): - monkeypatch.setenv("CLAUDE_PROJECT_DIR", project_dir) - monkeypatch.setenv("TOOL_INPUT", json.dumps(tool_input)) - - -def test_unapproved_edit_never_emits_a_memory_suggestion(tmp_path, monkeypatch, capsys): - (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "server.py"), - "old_string": "", - "new_string": "class UserAuthService:\n def __init__(self):\n pass\n", - }, - ) - - with pytest.raises(SystemExit) as exc_info: - main() - - assert exc_info.value.code == 0 - out = capsys.readouterr().out - assert out == "" - - -def test_trivial_change_no_output(tmp_path, monkeypatch, capsys): - (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "notes.txt"), - "old_string": "hello", - "new_string": "world", - }, +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +PROBE = ( + "import runpy, sys\n" + "try:\n" + " runpy.run_module('daem0nmcp.claude_hooks.post_edit', run_name='__main__')\n" + "except SystemExit as exc:\n" + " code = exc.code\n" + "print(code, 'daem0nmcp.edit_host' in sys.modules)\n" +) + + +def _run(event: dict, extra_env: dict | None = None) -> subprocess.CompletedProcess: + environment = {**os.environ, "PYTHONPATH": str(ROOT)} + environment.pop("DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", None) + environment.update(extra_env or {}) + return subprocess.run( + [sys.executable, "-c", PROBE], + input=json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, ) - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" +def _event(project: Path) -> dict: + return { + "session_id": "s-1", + "cwd": str(project), + "hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_use_id": "toolu_1", + "tool_input": {"file_path": str(project / "a.py"), "content": "x = 1\n"}, + "tool_response": {"success": True}, + } -def test_large_unapproved_change_is_not_captured_from_hook_text( - tmp_path, monkeypatch, capsys -): +def test_unpaired_project_exits_zero_without_importing_the_bridge(tmp_path): (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "big.py"), - "old_string": "", - "new_string": "x = 1\n" * 200, - }, - ) + result = _run(_event(tmp_path)) + assert result.returncode == 0, result.stderr + assert result.stdout.split() == ["0", "False"] - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" - - -def test_no_file_path_exits_clean(tmp_path, monkeypatch, capsys): +def test_paired_project_without_consumed_edit_exits_zero_silently(tmp_path): (tmp_path / ".daem0nmcp").mkdir() - _set_env(monkeypatch, str(tmp_path), {"command": "ls"}) - - with pytest.raises(SystemExit) as exc_info: - main() - - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" - - -def test_no_project_exits_clean(tmp_path, monkeypatch, capsys): - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "server.py"), - "old_string": "", - "new_string": "class Foo: pass", - }, + result = _run( + _event(tmp_path), + {"DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE": str(tmp_path / "missing.json")}, ) - - with pytest.raises(SystemExit) as exc_info: - main() - - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" + assert result.returncode == 0, result.stderr + assert result.stdout.split() == ["0", "True"] diff --git a/tests/test_claude_hooks_pre_edit.py b/tests/test_claude_hooks_pre_edit.py index 7987b59..558bc49 100644 --- a/tests/test_claude_hooks_pre_edit.py +++ b/tests/test_claude_hooks_pre_edit.py @@ -1,109 +1,121 @@ -"""Tests for the pre_edit Claude Code hook (blocking).""" - -import pytest -import pytest_asyncio - -from daem0nmcp.claude_hooks.pre_edit import async_main, handle_pre_edit -from daem0nmcp.database import DatabaseManager -from daem0nmcp.edit_bridge_transport import provision_bridge_credential - - -@pytest_asyncio.fixture -async def tmp_project(tmp_path): - """Create a temp project with initialised database.""" - daem0n_dir = tmp_path / ".daem0nmcp" - daem0n_dir.mkdir() - storage = daem0n_dir / "storage" - storage.mkdir() - - db = DatabaseManager(str(storage)) - await db.init_db() - yield tmp_path - await db.close() - - -@pytest.mark.asyncio -async def test_blocks_without_preflight(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" - assert str(tmp_project) not in result.message - - -@pytest.mark.asyncio -async def test_legacy_context_state_cannot_bypass_v7_preflight(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" - - -@pytest.mark.asyncio -async def test_permissive_mode_allows_through(tmp_project, monkeypatch): - """In permissive mode the block message is returned but allowed=False still.""" - monkeypatch.setenv("DAEM0N_HOOKS_PERMISSIVE", "1") +"""Tests for the pre_edit Claude Code hook (remind only, never block).""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _run(args: list[str], event: dict, env: dict | None = None): + environment = {**os.environ, "PYTHONPATH": str(ROOT)} + environment.pop("CLAUDE_PROJECT_DIR", None) + environment.update(env or {}) + return subprocess.run( + [sys.executable, *args], + input=json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, + ) - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - # async_main still returns allowed=False (it doesn't know about permissive mode) - # but the message is what block() would print. main() handles the permissive exit. - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +def _write_event(cwd: Path, file_path: str) -> dict: + return { + "session_id": "s-1", + "transcript_path": str(cwd / "t.jsonl"), + "cwd": str(cwd), + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"file_path": file_path, "content": "x = 1\n"}, + "tool_use_id": "toolu_1", + } -@pytest.mark.asyncio -async def test_does_not_read_or_surface_legacy_file_memories(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert "race condition" not in result.message - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +def test_non_daem0n_directory_exits_zero_silently(tmp_path): + result = _run( + ["-m", "daem0nmcp.claude_hooks.pre_edit"], + _write_event(tmp_path, str(tmp_path / "app.py")), + ) + assert result.returncode == 0 + assert result.stdout == "" -def test_no_file_path_exits_clean(tmp_path, monkeypatch): +def test_daem0n_project_exits_zero_with_reminder(tmp_path): (tmp_path / ".daem0nmcp").mkdir() - monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path)) - monkeypatch.setenv("TOOL_INPUT", '{"command": "ls -la"}') + result = _run( + ["-m", "daem0nmcp.claude_hooks.pre_edit"], + _write_event(tmp_path, str(tmp_path / "src" / "app.py")), + ) + assert result.returncode == 0 + payload = json.loads(result.stdout) + output = payload["hookSpecificOutput"] + assert output["hookEventName"] == "PreToolUse" + assert "permissionDecision" not in output + assert "memory_preflight" in output["additionalContext"] + assert ( + 'memory_recall_file(relative_file_path="src/app.py")' + in output["additionalContext"] + ) + assert "\n" not in output["additionalContext"] - from daem0nmcp.claude_hooks.pre_edit import main - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 2 +def test_falls_back_to_claude_project_dir(tmp_path): + (tmp_path / ".daem0nmcp").mkdir() + event = _write_event(tmp_path, "notes.md") + del event["cwd"] + result = _run( + ["-m", "daem0nmcp.claude_hooks.pre_edit"], + event, + env={"CLAUDE_PROJECT_DIR": str(tmp_path)}, + ) + assert result.returncode == 0 + context = json.loads(result.stdout)["hookSpecificOutput"]["additionalContext"] + assert 'relative_file_path="notes.md"' in context -def test_remote_mode_without_workspace_binding_fails_recoverably(tmp_path, monkeypatch): - target = tmp_path / "target.txt" - target.write_text("before", encoding="utf-8") - credential = tmp_path / "host" / "credential.json" - provision_bridge_credential( - credential, - principal_id="remote-principal", - transports=frozenset({"remote-https"}), +def test_path_outside_project_and_malformed_input_still_exit_zero(tmp_path): + (tmp_path / "project" / ".daem0nmcp").mkdir(parents=True) + outside = _run( + ["-m", "daem0nmcp.claude_hooks.pre_edit"], + _write_event(tmp_path / "project", str(tmp_path / "elsewhere.py")), ) - ca_file = tmp_path / "ca.pem" - ca_file.write_text("unused", encoding="utf-8") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", str(credential)) - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_MODE", "remote-https") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_REMOTE_URL", "https://server.example") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_CA_FILE", str(ca_file)) - monkeypatch.delenv("DAEM0NMCP_EDIT_HOST_WORKSPACE_BINDING_FILE", raising=False) - - result = handle_pre_edit( - { - "session_id": "remote-session", - "tool_use_id": "remote-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "before", - "new_string": "after", - }, - }, - str(tmp_path), + assert outside.returncode == 0 + context = json.loads(outside.stdout)["hookSpecificOutput"]["additionalContext"] + assert "relative_file_path" not in context + + environment = {**os.environ, "PYTHONPATH": str(ROOT)} + garbage = subprocess.run( + [sys.executable, "-m", "daem0nmcp.claude_hooks.pre_edit"], + input="not json", + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, ) + assert garbage.returncode == 0 + assert garbage.stdout == "" + - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +def test_imports_no_api_or_retrieval_modules(tmp_path): + (tmp_path / ".daem0nmcp").mkdir() + probe = ( + "import sys, runpy\n" + "try:\n" + " runpy.run_module('daem0nmcp.claude_hooks.pre_edit', run_name='__main__')\n" + "except SystemExit:\n" + " pass\n" + "heavy = sorted(m for m in sys.modules if m.startswith((" + "'daem0nmcp.api', 'daem0nmcp.retrieval', 'daem0nmcp.edit_host'," + " 'daem0nmcp.database', 'sqlalchemy', 'pydantic')))\n" + "print(heavy, file=sys.stderr)\n" + "sys.exit(1 if heavy else 0)\n" + ) + result = _run(["-c", probe], _write_event(tmp_path, str(tmp_path / "a.py"))) + assert result.returncode == 0, result.stderr + assert "additionalContext" in result.stdout diff --git a/tests/test_claude_hooks_stop.py b/tests/test_claude_hooks_stop.py index adce7a2..1b585fd 100644 --- a/tests/test_claude_hooks_stop.py +++ b/tests/test_claude_hooks_stop.py @@ -1,5 +1,11 @@ """Tests for the stop Claude Code hook.""" +import json +import os +import subprocess +import sys +from pathlib import Path + import pytest import pytest_asyncio @@ -108,3 +114,89 @@ async def test_no_messages_returns_empty(tmp_project): # 0-length messages -> anti-loop sees turn 0 vs -1, which passes, # but no content means no completion signal -> empty assert result.message == "" + + +def _run_stop(event: dict, home: Path) -> subprocess.CompletedProcess[str]: + environment = { + **os.environ, + "PYTHONPATH": str(Path(__file__).resolve().parents[1]), + "HOME": str(home), + "USERPROFILE": str(home), + } + environment.pop("CLAUDE_PROJECT_DIR", None) + return subprocess.run( + [sys.executable, "-m", "daem0nmcp.claude_hooks.stop"], + input=json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, + ) + + +def test_main_reads_stdin_event_and_nested_transcript(tmp_path): + project = tmp_path / "project" + (project / ".daem0nmcp").mkdir(parents=True) + transcript = tmp_path / "transcript.jsonl" + records = [ + {"type": "summary", "summary": "Caching work"}, + { + "type": "user", + "sessionId": "s-1", + "message": {"role": "user", "content": "Add caching"}, + }, + { + "type": "assistant", + "sessionId": "s-1", + "message": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": ( + "I will use Redis for caching because it gives shared " + "state. Implementation is complete and all tasks are done." + ), + } + ], + }, + }, + ] + transcript.write_text( + "\n".join(json.dumps(record) for record in records) + "\n", encoding="utf-8" + ) + event = { + "session_id": "s-1", + "transcript_path": str(transcript), + "cwd": str(project), + "hook_event_name": "Stop", + "stop_hook_active": False, + } + + result = _run_stop(event, tmp_path) + + assert result.returncode == 0, result.stderr + assert '"decision"' not in result.stdout + message = json.loads(result.stdout)["systemMessage"] + assert "mcp__daem0nmcp__memory_store" in message + assert "memory_record_outcome" in message + assert not (project / ".daem0nmcp" / "storage").exists() + assert (tmp_path / ".daem0nmcp" / "hook_state" / "stop_s-1.json").is_file() + + +def test_main_is_silent_outside_a_daem0n_project(tmp_path): + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + json.dumps( + {"message": {"role": "assistant", "content": "All tasks are complete."}} + ) + + "\n", + encoding="utf-8", + ) + result = _run_stop( + {"session_id": "s-2", "transcript_path": str(transcript), "cwd": str(tmp_path)}, + tmp_path, + ) + assert result.returncode == 0 + assert result.stdout == "" diff --git a/tests/test_edit_bridge_transport.py b/tests/test_edit_bridge_transport.py index 15c0e77..a574d17 100644 --- a/tests/test_edit_bridge_transport.py +++ b/tests/test_edit_bridge_transport.py @@ -27,7 +27,6 @@ from daem0nmcp.claude_hooks.post_edit_preflight import ( handle_edit_preflight_response, ) -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.covenant import InvocationScope from daem0nmcp.database import DatabaseManager from daem0nmcp.edit_bridge import ( @@ -51,6 +50,7 @@ provision_remote_bridge_installation, ) from daem0nmcp.workspace import WorkspaceRegistry +from tests.native_edit_host import drive_native_edit SECRET = "bridge-secret-" + "x" * 48 AUTHKEY = b"local-auth-key-" + b"y" * 32 @@ -510,8 +510,8 @@ async def test_remote_hooks_bind_different_desktop_root_to_server_workspace( } try: with patch.dict(os.environ, environment, clear=False): - denied = handle_pre_edit(first_event, str(client_root)) - assert not denied.allowed + denied = drive_native_edit(first_event, str(client_root)) + assert not denied assert ( workspace.workspace_id != WorkspaceRegistry(default_root=client_root).default.workspace_id @@ -567,8 +567,8 @@ async def test_remote_hooks_bind_different_desktop_root_to_server_workspace( ) retry_event = {**first_event, "tool_use_id": "remote-allowed-2"} with patch.dict(os.environ, environment, clear=False): - allowed = handle_pre_edit(retry_event, str(client_root)) - assert allowed.allowed + allowed = drive_native_edit(retry_event, str(client_root)) + assert allowed target.write_text("after", encoding="utf-8") with patch.dict(os.environ, environment, clear=False): assert handle_post_edit(retry_event, str(client_root)) diff --git a/tests/test_edit_host.py b/tests/test_edit_host.py index 8a5e2a0..3243414 100644 --- a/tests/test_edit_host.py +++ b/tests/test_edit_host.py @@ -11,7 +11,6 @@ import pytest -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.edit_bridge_transport import ( RemoteBridgeHTTPSClient, local_bridge_address, @@ -29,6 +28,7 @@ ) from daem0nmcp.protected_files import verify_owner_only_file from daem0nmcp.workspace import WorkspaceRegistry +from tests.native_edit_host import drive_native_edit def _configuration(tmp_path): @@ -236,21 +236,20 @@ def unexpected_call(*_args, **_kwargs): with monkeypatch.context() as context: for name, value in installation.environment().items(): context.setenv(name, value) - result = handle_pre_edit( - { - "session_id": "replacement-session", - "tool_use_id": "replacement-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "replacement", - "new_string": "changed", + with pytest.raises(ValueError): + drive_native_edit( + { + "session_id": "replacement-session", + "tool_use_id": "replacement-request", + "tool_name": "Edit", + "tool_input": { + "file_path": str(target), + "old_string": "replacement", + "new_string": "changed", + }, }, - }, - str(project), - ) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" + str(project), + ) assert calls == 0 project.rename(tmp_path / "replacement-checkout") (tmp_path / "original-checkout").rename(project) @@ -317,21 +316,20 @@ def unexpected_call(*_args, **_kwargs): with monkeypatch.context() as context: for name, value in environment.items(): context.setenv(name, value) - result = handle_pre_edit( - { - "session_id": "substitution-session", - "tool_use_id": "substitution-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "before", - "new_string": "after", + with pytest.raises(ValueError): + drive_native_edit( + { + "session_id": "substitution-session", + "tool_use_id": "substitution-request", + "tool_name": "Edit", + "tool_input": { + "file_path": str(target), + "old_string": "before", + "new_string": "after", + }, }, - }, - str(project), - ) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" + str(project), + ) assert calls == 0 with pytest.raises(ValueError, match="conflicts"): From a9e7d7a65c8d1126e3467fc2fdb6b9d4587d17b9 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:00:56 -0400 Subject: [PATCH 21/72] docs: describe the six remind-only Claude Code hooks and pairing Replace the hook sections of the README, Summon section 9 and the v7 operations guide with one table of the installed hooks, the user-level install location, and what pairing writes. Delete the README manual hooks block and tell manual-hook users to re-run install-claude-hooks. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 5 +++ README.md | 71 ++++++++++++++++++++----------------------- Summon_Daem0n.md | 30 +++++++++++++----- docs/v7-operations.md | 27 ++++++++++++++++ 4 files changed, 88 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c67c0..06c7750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ ### Added - Reproducible v7 core foundation with opt-in dependency profiles and lazy capability reporting. +- `uninstall-claude-hooks` with `--project-path` strips the pairing env keys from the project's `.claude/settings.local.json`; `--remove-credentials` also deletes its local edit-bridge credential and socket directories. + +### Changed +- Claude Code hooks remind instead of block. `pre_edit` adds a one-line `memory_recall_file`/`memory_preflight` reminder to the model's context and never denies an edit; `stop` reads its stdin event, shows suggestions as a `systemMessage` and never keeps the agent running. +- The manual `hooks/daem0n_*.py` scripts are deprecated stubs that print a notice and exit 0, and `hooks/settings.json.example` is removed. If you installed them by hand, re-run `python -m daem0nmcp.cli install-claude-hooks`; it also removes the old entries. ## [6.0.0] - 2026-01-29 diff --git a/README.md b/README.md index daad047..728f2da 100644 --- a/README.md +++ b/README.md @@ -608,55 +608,50 @@ Use `system_health` for bounded diagnostics. The complete protocol lives in ## Claude Code Integration -### Automated Hook Installation (Recommended) +### Hook Installation ```bash python -m daem0nmcp.cli install-claude-hooks ``` -This registers 5 hook modules in `~/.claude/settings.json`: +This writes six hook entries to the user-level `~/.claude/settings.json`, so +they run in every project; each runs `"" -m daem0nmcp.claude_hooks.` +with the interpreter you installed from, and stays silent in directories without +`.daem0nmcp/`. Other hooks in that file are kept, and any old +`hooks/daem0n_*.py` entries are removed. -| Event | Hook | Purpose | -|-------|------|---------| -| `SessionStart` | `session_start` | Scoped `session_brief` instruction | -| `PreToolUse` (Edit/Write/NotebookEdit) | `pre_edit` | Fail-closed v7 preflight instruction | -| `PreToolUse` (Bash) | `pre_bash` | Rule enforcement on commands | -| `PostToolUse` (Edit/Write) | `post_edit` | Replay-safe memory suggestion | -| `Stop`/`SubagentStop` | `stop` | Outcome reminder without direct memory writes | +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | -To remove: `python -m daem0nmcp.cli uninstall-claude-hooks` +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. -### Manual Hooks (Legacy) +**What pairing writes.** The installer also pairs the current project (or +`--project-path`) with the local edit bridge: -Add to `.claude/settings.json`: -```json -{ - "hooks": { - "PreToolUse": [{ - "matcher": "Edit|Write|NotebookEdit", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_pre_edit_hook.py\"" - }] - }], - "PostToolUse": [{ - "matcher": "Edit|Write", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_post_edit_hook.py\"" - }] - }], - "Stop": [{ - "matcher": "", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - }] - }] - } -} +- an owner-only credential at `~/.daem0nmcp/edit-bridges//credential.json` + (the secret is never written into the project), with the bridge socket under + `~/.daem0nmcp/edit-bridges/run//`; +- the env keys `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, + `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, `DAEM0NMCP_PROJECT_ROOT` and + `DAEM0NMCP_STORAGE_PATH` in `/.claude/settings.local.json`. + +To remove the hooks, and optionally the project's pairing and credential: + +```bash +python -m daem0nmcp.cli --project-path uninstall-claude-hooks [--remove-credentials] ``` +The old manual scripts in `hooks/` are deprecated stubs that only print a +notice and exit 0. If your settings still point at them, re-run +`install-claude-hooks`. + ### Protocol Skill A Claude Code skill is included at `.claude/skills/daem0nmcp-protocol/SKILL.md` that enforces the memory protocol automatically. diff --git a/Summon_Daem0n.md b/Summon_Daem0n.md index eafc434..379a457 100644 --- a/Summon_Daem0n.md +++ b/Summon_Daem0n.md @@ -167,18 +167,34 @@ tools, never through a direct database, script, or memory-writing CLI command. ## 9. Hook behavior -The root hook templates are advisory and read-only. The pre-edit hook fails -closed because a standalone process cannot validate a scoped capability. It -directs Claude Code to the exact `memory_recall` and `memory_preflight` calls. -Post-edit and Stop hooks print replay-safe `memory_store` and -`memory_record_outcome` suggestions; they do not mutate Daem0n storage. - -Install the packaged hook configuration with: +Install the packaged hooks with: ```bash python -m daem0nmcp.cli install-claude-hooks ``` +This writes six entries to the user-level `~/.claude/settings.json`. The hooks +remind; they never block an edit, a command, or the end of a turn, and none of +them writes Daem0n memory. + +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | + +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. + +Run from a project, the installer also pairs that project with the local edit +bridge: an owner-only credential under `~/.daem0nmcp/edit-bridges//` +and bridge path env keys in `/.claude/settings.local.json`. The old +`hooks/daem0n_*.py` scripts are deprecated stubs; re-run `install-claude-hooks` +if your settings still point at them. + ## 10. Migration reference The generated source of truth for renamed or split v6 capabilities is diff --git a/docs/v7-operations.md b/docs/v7-operations.md index ba19798..16e6b70 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -257,6 +257,33 @@ Install Claude hooks for one project: python -m daem0nmcp.cli --project-path /srv/project-a install-claude-hooks ``` +The hooks go into the user-level `~/.claude/settings.json` and run in every +project, staying silent where there is no `.daem0nmcp/`. They remind only: + +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | + +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. + +Pairing writes an owner-only credential to +`~/.daem0nmcp/edit-bridges//credential.json`, uses +`~/.daem0nmcp/edit-bridges/run//` for the bridge socket, and +adds `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, +`DAEM0NMCP_PROJECT_ROOT` and `DAEM0NMCP_STORAGE_PATH` to +`/.claude/settings.local.json`. Remove the hooks, those env keys and +(with `--remove-credentials`) the credential and socket directories with: + +```bash +python -m daem0nmcp.cli --project-path /srv/project-a uninstall-claude-hooks --remove-credentials +``` + Install OpenCode V1 locally: ```bash From 358256fb953692448e24177bef1f1af176dc65b4 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:03:56 -0400 Subject: [PATCH 22/72] fix: report only SQLite integrity errors as event stream conflicts Any INSERT failure (I/O, lock) used to become a retryable EVENT_STREAM_CONFLICT. Only sqlite3.IntegrityError is a real conflict; everything else now propagates to the caller's own mapping (F-066). Co-Authored-By: Claude Opus 5 --- daem0nmcp/event_store.py | 6 +++++- tests/test_event_store.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/daem0nmcp/event_store.py b/daem0nmcp/event_store.py index bfea017..fd11307 100644 --- a/daem0nmcp/event_store.py +++ b/daem0nmcp/event_store.py @@ -11,6 +11,7 @@ import json import math import re +import sqlite3 import unicodedata from collections.abc import Iterator, Mapping from dataclasses import dataclass @@ -536,7 +537,10 @@ def _append(self, command: EventCommand, payload: dict[str, Any]) -> AppendedEve """, values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # Only a uniqueness/constraint collision is a stream conflict; I/O + # and lock errors propagate so callers do not advise a retry of a + # conflict that never happened. raise EventStreamConflict( "event identity or stream version is occupied" ) from exc diff --git a/tests/test_event_store.py b/tests/test_event_store.py index 5d2144d..df75ba4 100644 --- a/tests/test_event_store.py +++ b/tests/test_event_store.py @@ -257,6 +257,43 @@ def test_expected_version_is_exactly_idempotent_or_conflicts(self): self.connection.execute("SELECT count(*) FROM memory_events").fetchone()[0], ) + def test_only_integrity_errors_on_insert_are_stream_conflicts(self): + """A lock or I/O failure must not be reported as a retryable conflict.""" + command_type, store_type = self._api() + from daem0nmcp.event_store import EventStreamConflict + + connection = self.connection + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO memory_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = command_type( + workspace_id=self.workspace_id, + stream_id="mem_" + "c" * 64, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + payload={"record": self._state()}, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + store_type(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + store_type(failing).append_and_project(command) + connection.rollback() + def test_projection_failure_rolls_back_event_with_savepoint(self): """No immutable event may survive a failed projection in caller transaction.""" command_type, store_type = self._api() From 83a1b7bfe1d4468585806ca89ec61890366605a0 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:03:56 -0400 Subject: [PATCH 23/72] fix: report transient contention as retryable and log every internal error - A full writer/resource worker pool or a SQLite "database is locked"/busy timeout now surfaces as DATABASE_IN_USE (retryable, retry_after_ms=250) instead of a non-retryable INTERNAL_ERROR (F-020, F-027, A-011). This covers memory_store, memory_record_outcome, session_brief and memory_preflight through one is_database_busy() classifier in errors.py. - ResponseContext.internal_error logs the cause under the correlation ID the client receives (stderr via logging, never the MCP wire; the envelope is unchanged). Runtime service errors chain their cause (`from exc`) so the log shows the real failure (F-021, F-028). - memory_preflight with target arguments that do not validate stays ok:true but adds a PREFLIGHT_TOKEN_NOT_ISSUED warning naming the failing fields (field names and pydantic error types only) (F-070, F-082). - Tests: 8 parallel preflight+store flows, a held write lock, a logged induced failure, a full resource pool, and two stdio server processes firing 5 same-key and 10 distinct-key stores (F-055). Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/errors.py | 20 ++ daem0nmcp/api/v7/pinned.py | 62 +++++- daem0nmcp/api/v7/resource_repository.py | 16 +- daem0nmcp/api/v7/responses.py | 16 +- daem0nmcp/api/v7/runtime_services.py | 119 ++++++----- tests/api_v7/process_client.py | 3 +- tests/api_v7/test_error_honesty.py | 192 ++++++++++++++++++ tests/api_v7/test_pinned_handlers.py | 48 +++++ .../api_v7/test_process_concurrent_writers.py | 97 +++++++++ tests/api_v7/test_runtime_services.py | 5 +- 10 files changed, 510 insertions(+), 68 deletions(-) create mode 100644 tests/api_v7/test_error_honesty.py create mode 100644 tests/api_v7/test_process_concurrent_writers.py diff --git a/daem0nmcp/api/v7/errors.py b/daem0nmcp/api/v7/errors.py index c42fe14..b0e5820 100644 --- a/daem0nmcp/api/v7/errors.py +++ b/daem0nmcp/api/v7/errors.py @@ -6,9 +6,12 @@ from __future__ import annotations +import sqlite3 from enum import Enum from types import MappingProxyType +from ...bounded_workers import BoundedWorkerBusyError + class ErrorCode(str, Enum): """Reviewed business-error codes returned inside ``ApiResponse``.""" @@ -63,11 +66,28 @@ def is_stable_error_code(value: object) -> bool: return isinstance(value, str) and value in STABLE_ERROR_CODE_SET +def is_database_busy(error: BaseException | None) -> bool: + """Return whether *error* is transient contention a caller may retry. + + A full worker pool and a SQLite lock or busy timeout both clear once the + competing writer finishes, so they map to retryable ``DATABASE_IN_USE``. + """ + + if isinstance(error, BoundedWorkerBusyError): + return True + if getattr(error, "code", None) == ErrorCode.DATABASE_IN_USE.value: + return True + return isinstance(error, sqlite3.OperationalError) and any( + word in str(error).casefold() for word in ("locked", "busy") + ) + + __all__ = [ "ERROR_CODE_REGISTRY", "INTERNAL_ERROR_MESSAGE", "STABLE_ERROR_CODES", "STABLE_ERROR_CODE_SET", "ErrorCode", + "is_database_busy", "is_stable_error_code", ] diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 9b612ba..b03bdb9 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -11,6 +11,8 @@ from types import MappingProxyType from typing import Any, Protocol, TypeVar +from pydantic import ValidationError + from ...covenant import ( ArgumentNormalizationError, CovenantGate, @@ -21,7 +23,13 @@ from ...retrieval import RetrievalQuery from ...workspace import Workspace from .errors import ErrorCode -from .models import ApiResponse, CapabilityState, RecordSummary, RetrievalData +from .models import ( + ApiResponse, + ApiWarning, + CapabilityState, + RecordSummary, + RetrievalData, +) from .responses import ResponseContext, ResponseFactory from .tasks import ( durable_task_execution_var, @@ -104,10 +112,46 @@ def _contains_raw_path(value: object) -> bool: T = TypeVar("T") -def _path_safe_success(response: ResponseContext, data: T) -> ApiResponse[T]: +def _path_safe_success( + response: ResponseContext, + data: T, + *, + warnings: list[ApiWarning] | None = None, +) -> ApiResponse[T]: if _contains_raw_path(data): return response.internal_error() - return response.success(data) + return response.success(data, warnings=warnings or ()) + + +_FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") + + +def _token_not_issued_warning(error: ArgumentNormalizationError) -> ApiWarning: + """Say why a description-only preflight carries no token. + + Only schema field names and pydantic error types are echoed; supplied + values and unknown keys never reach the wire. + """ + + problems: list[str] = [] + cause = error.__cause__ + if isinstance(cause, ValidationError): + for item in cause.errors(): + location = ".".join( + str(part) + if isinstance(part, int) or _FIELD_NAME.match(str(part)) + else "?" + for part in item["loc"] + ) + problems.append(f"{location or 'target_arguments'} ({item['type']})") + detail = "; ".join(problems)[:360] or "they do not match the target tool schema" + return ApiWarning( + code="PREFLIGHT_TOKEN_NOT_ISSUED", + message=( + f"Target arguments did not validate: {detail}. " + "No preflight token issued; fix them and call memory_preflight again." + ), + ) _EXPECTED_SERVICE_ERRORS = MappingProxyType( @@ -192,7 +236,12 @@ def _expected_service_failure( if mapped is None: return None code, message, retryable = mapped - return response.failure(code, message, retryable=retryable) + return response.failure( + code, + message, + retryable=retryable, + retry_after_ms=250 if code is ErrorCode.DATABASE_IN_USE else None, + ) def _utc_now() -> datetime: @@ -507,13 +556,14 @@ async def memory_preflight( **request.target_arguments, } target_is_complete = True + draft_warning: ApiWarning | None = None try: normalized_arguments = self._dependencies.argument_normalizer( request.target_tool, target_call_arguments, scope.canonical_workspace, ) - except ArgumentNormalizationError: + except ArgumentNormalizationError as normalization_error: if request.description is None: return response.failure( ErrorCode.INVALID_ARGUMENT, @@ -524,6 +574,7 @@ async def memory_preflight( # can never become capability material until full normalization # succeeds. target_is_complete = False + draft_warning = _token_not_issued_warning(normalization_error) normalized_arguments = dict(request.target_arguments) try: guidance_value = self._dependencies.preflight_service.guidance( @@ -549,6 +600,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=None, ), + warnings=[draft_warning] if draft_warning is not None else None, ) token = self._dependencies.covenant_gate.issue_preflight( scope, diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index c1c7a07..d6b839c 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -30,6 +30,7 @@ from ...schema_version import CURRENT_SCHEMA_VERSION from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path +from .errors import is_database_busy from .models import RecordSummary from .public_ids import PublicObjectIdRepository from .resources import ( @@ -209,8 +210,9 @@ class ResourceRepositoryError(RuntimeError): code = "RESOURCE_REPOSITORY_UNAVAILABLE" - def __init__(self) -> None: - super().__init__(self.code) + def __init__(self, code: str = "RESOURCE_REPOSITORY_UNAVAILABLE") -> None: + self.code = code + super().__init__(code) ActiveDatabaseResolver = Callable[[Workspace], ResolvedActiveDatabase] @@ -978,12 +980,14 @@ async def _run(self, operation: Callable[[], T]) -> T: continue except Exception: break - if worker.done(): - with suppress(Exception): - worker.result() + failure: BaseException | None = None + if worker.done() and not worker.cancelled(): + failure = worker.exception() if cancellation is not None: raise cancellation - raise ResourceRepositoryError() + if is_database_busy(failure): + raise ResourceRepositoryError("DATABASE_IN_USE") from failure + raise ResourceRepositoryError() from failure def _open_connection( self, diff --git a/daem0nmcp/api/v7/responses.py b/daem0nmcp/api/v7/responses.py index fa51258..241b656 100644 --- a/daem0nmcp/api/v7/responses.py +++ b/daem0nmcp/api/v7/responses.py @@ -2,7 +2,9 @@ from __future__ import annotations +import logging import secrets +import sys from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone @@ -22,6 +24,8 @@ T = TypeVar("T") +_LOGGER = logging.getLogger(__name__) + def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -118,9 +122,17 @@ def failure( ) def internal_error(self, error: BaseException | None = None) -> ApiResponse[Any]: - """Return the one deliberately opaque internal failure envelope.""" + """Return the one deliberately opaque internal failure envelope. + + The cause is logged (stderr by default, never the MCP wire) under the + correlation ID the caller receives, so the owner can find it. + """ - del error + _LOGGER.error( + "v7 internal error correlation_id=%s", + self.request_id, + exc_info=error if error is not None else sys.exc_info()[1], + ) return self.failure( ErrorCode.INTERNAL_ERROR, INTERNAL_ERROR_MESSAGE, diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index aada0f8..dd24caa 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -49,6 +49,7 @@ normalize_resolved_path, resolve_derived_path, ) +from .errors import is_database_busy from .federated_retrieval import ( FederatedCandidate, FederatedRetrievalError, @@ -145,8 +146,8 @@ def _validated_workspace(workspace: Workspace) -> Workspace: try: canonical = normalize_resolved_path(workspace.root.resolve(strict=True)) registered = WorkspaceRegistry([canonical], default_root=canonical).default - except (OSError, RuntimeError, TypeError, ValueError): - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except (OSError, RuntimeError, TypeError, ValueError) as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if ( registered.workspace_id != workspace.workspace_id or os.path.normcase(str(registered.root)) != os.path.normcase(str(canonical)) @@ -167,8 +168,8 @@ def resolve_workspace_storage(workspace: Workspace) -> Path: ".daem0nmcp", "storage", ) - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc if not storage.is_dir() or storage.is_symlink(): raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") return storage @@ -194,9 +195,9 @@ def locked_current( except RuntimeServiceError: lock.release() raise - except Exception: + except Exception as exc: lock.release() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc try: yield active finally: @@ -264,10 +265,12 @@ def _open_database(path: Path, *, writable: bool) -> sqlite3.Connection: with suppress(NameError, sqlite3.Error): connection.close() raise - except Exception: + except Exception as exc: with suppress(NameError, sqlite3.Error): connection.close() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc def _inspect_database_health( @@ -319,8 +322,8 @@ def _inspect_database_health( # structurally incomplete storage. Full-page corruption certification # remains the explicit offline verify-v7 operation. return schema_version, tables - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc finally: if connection is not None: with suppress(sqlite3.Error): @@ -335,8 +338,8 @@ def _datetime_us(value: datetime) -> int: epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) delta = utc - epoch result = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds - except (OverflowError, ValueError): - raise RuntimeServiceError("INVALID_TIMESTAMP") from None + except (OverflowError, ValueError) as exc: + raise RuntimeServiceError("INVALID_TIMESTAMP") from exc if not -(2**63) <= result <= 2**63 - 1: raise RuntimeServiceError("INVALID_TIMESTAMP") return result @@ -347,8 +350,8 @@ def _datetime_from_us(value: object) -> datetime: raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") try: return datetime.fromtimestamp(value / 1_000_000, timezone.utc) - except (OverflowError, OSError, ValueError): - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except (OverflowError, OSError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: @@ -364,8 +367,8 @@ def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: def _parse_json(value: object, expected_type: type, code: str) -> Any: try: parsed = json.loads(str(value)) - except (TypeError, ValueError, RecursionError): - raise RuntimeServiceError(code) from None + except (TypeError, ValueError, RecursionError) as exc: + raise RuntimeServiceError(code) from exc if not isinstance(parsed, expected_type): raise RuntimeServiceError(code) return parsed @@ -384,8 +387,8 @@ def _event_receipt(row: sqlite3.Row) -> AppendedEvent: else str(row["previous_event_hash"]) ), ) - except (KeyError, TypeError, ValueError): - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc def _idempotency_correlation( @@ -427,8 +430,8 @@ def _existing_idempotent_event( payload = _parse_json(row["payload_json"], dict, "MEMORY_EVENT_INTEGRITY_FAILED") try: canonical = canonical_json_bytes(payload).decode("utf-8") - except Exception: - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc if canonical != str(row["payload_json"]) or sha256_json(payload) != str( row["payload_hash"] ): @@ -494,8 +497,8 @@ def _record_summary( ) except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_state(row: sqlite3.Row) -> dict[str, Any]: @@ -585,6 +588,8 @@ async def _run_mutation( worker = asyncio.create_task(self._workers.run(lambda: operation(cancelled))) try: return await asyncio.shield(worker) + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("DATABASE_IN_USE") from exc except asyncio.CancelledError as cancellation: cancelled.set() try: @@ -598,8 +603,8 @@ async def _run_mutation( def _now_us(self) -> int: try: value = self._clock() - except Exception: - raise RuntimeServiceError("CLOCK_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("CLOCK_UNAVAILABLE") from exc return _datetime_us(value) def _schedule_after_commit(self, path: Path, changed: bool) -> None: @@ -760,10 +765,12 @@ def _store_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_STORE_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_STORE_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -880,10 +887,12 @@ def _record_outcome_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -949,8 +958,8 @@ def _retrieval_config_fingerprint( "capabilities": dict(sorted(capability_statuses.items())), } ) - except (TypeError, ValueError): - raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from None + except (TypeError, ValueError) as exc: + raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from exc @dataclass(slots=True) @@ -1084,8 +1093,8 @@ async def _run_snapshot(self, operation: Callable[[], Any]) -> Any: worker = asyncio.create_task(self._workers.run(operation)) try: return await asyncio.shield(worker) - except BoundedWorkerBusyError: - raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from None + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from exc except asyncio.CancelledError: with suppress(asyncio.CancelledError, Exception): await await_task_terminal(worker) @@ -1132,8 +1141,8 @@ async def retrieve( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc if _contains_raw_path(hydrated): raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return hydrated @@ -1167,8 +1176,8 @@ def _resolve_federated_workspace(self, workspace_id: str) -> Workspace: try: workspace = resolve(workspace_id) validated = _validated_workspace(workspace) - except Exception: - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except Exception as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if validated.workspace_id != workspace_id: raise RuntimeServiceError("INVALID_WORKSPACE") return validated @@ -1258,10 +1267,10 @@ async def retrieve_one( raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") finally: guard.release() - except asyncio.TimeoutError: - raise RuntimeServiceError("DEADLINE_EXCEEDED") from None + except asyncio.TimeoutError as exc: + raise RuntimeServiceError("DEADLINE_EXCEEDED") from exc except FederatedRetrievalError as exc: - raise RuntimeServiceError(exc.code) from None + raise RuntimeServiceError(exc.code) from exc except asyncio.CancelledError: raise except RuntimeServiceError: @@ -1272,8 +1281,8 @@ async def retrieve_one( "COMMUNION_REQUIRED", "UNAUTHORIZED_WORKSPACE", }: - raise RuntimeServiceError(authorization_code) from None - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + raise RuntimeServiceError(authorization_code) from exc + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc # `hydrated` is immutable and there is no await between releasing the # read guard above and committing this response to the caller. return hydrated @@ -1373,10 +1382,10 @@ def _hydrate_candidates( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1428,8 +1437,8 @@ def _hydrate( dropped=0, ), ) - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc if result.context is None: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") @@ -1541,10 +1550,10 @@ def _hydrate( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1565,8 +1574,8 @@ def _public_ref(ref: EvidenceRef, origin_workspace_id: str) -> PublicEvidenceRef relation_path=list(ref.relation_path), provider=provider, ) - except Exception: - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc class BriefingReader(Protocol): @@ -1620,8 +1629,10 @@ async def assemble( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("BRIEFING_FAILED") from None + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("BRIEFING_FAILED") from exc if data.workspace_id != workspace.workspace_id: raise RuntimeServiceError("BRIEFING_FAILED") if _contains_raw_path(data): @@ -1678,8 +1689,10 @@ async def guidance( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("PREFLIGHT_FAILED") from None + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("PREFLIGHT_FAILED") from exc if _contains_raw_path(data): raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index 1f2f795..e31e2f9 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -160,6 +160,7 @@ async def process_client( environment_overrides: dict[str, str] | None = None, http_probe: Callable[[str], Awaitable[None]] | None = None, http_headers: dict[str, str] | None = None, + log_name: str = "stdio-server.log", ): """Launch the installed module in a workspace and clean up on every exit.""" environment = server_environment( @@ -176,7 +177,7 @@ async def process_client( ) body_error: BaseException | None = None try: - with (workspace / "stdio-server.log").open("w", encoding="utf-8") as log: + with (workspace / log_name).open("w", encoding="utf-8") as log: async with ( stdio_client(parameters, errlog=log) as (read, write), ClientSession( diff --git a/tests/api_v7/test_error_honesty.py b/tests/api_v7/test_error_honesty.py new file mode 100644 index 0000000..26b9743 --- /dev/null +++ b/tests/api_v7/test_error_honesty.py @@ -0,0 +1,192 @@ +"""Transient contention is retryable and every internal error is logged (KD-6).""" + +from __future__ import annotations + +import asyncio +import json +import logging +import sqlite3 +import threading +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client + +from daem0nmcp.api.v7 import runtime_services +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resource_repository import ( + ResourceRepositoryError, + SQLiteResourceRepository, +) +from daem0nmcp.api.v7.resources import ResourceReadRequest +from daem0nmcp.bounded_workers import BoundedWorkerPool +from daem0nmcp.config import Settings +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces + +FLOWS = 8 + + +def _server(root: Path): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(root), + workspace_roots=[str(root)], + dream_enabled=False, + ), + environ={}, + ) + + +def _database(root: Path) -> Path: + return resolve_active_database(root / ".daem0nmcp" / "storage").path + + +def _is_retryable_busy(response: dict[str, Any]) -> bool: + error = response["error"] + return ( + error["code"] == "DATABASE_IN_USE" + and error["retryable"] is True + and error["retry_after_ms"] == 250 + ) + + +class _Session: + def __init__(self, client: Client, workspace_id: str) -> None: + self._client = client + self.workspace_id = workspace_id + + async def call(self, tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await self._client.call_tool( + tool, + {"workspace_id": self.workspace_id, **arguments}, + raise_on_error=False, + ) + assert isinstance(result.structured_content, dict), tool + return result.structured_content + + async def preflight_and_store(self, index: int) -> dict[str, Any]: + arguments = { + "record_type": "decision", + "content": f"Concurrent decision number {index}", + "idempotency_key": f"concurrent-store-{index:04d}", + } + preflight = await self.call( + "memory_preflight", + {"target_tool": "memory_store", "target_arguments": arguments}, + ) + if not preflight["ok"]: + return preflight + return await self.call( + "memory_store", + {**arguments, "preflight_token": preflight["data"]["preflight_token"]}, + ) + + +async def test_parallel_stores_are_ok_or_retryable_and_land_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + + async def flow(index: int) -> list[dict[str, Any]]: + observed = [] + deadline = asyncio.get_running_loop().time() + 60 + while True: + response = await session.preflight_and_store(index) + observed.append(response) + if response["ok"]: + return observed + assert _is_retryable_busy(response), response + assert asyncio.get_running_loop().time() < deadline, observed + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + results = await asyncio.gather(*(flow(index) for index in range(FLOWS))) + + for observed in results: + assert observed[-1]["ok"], observed + with closing(sqlite3.connect(_database(tmp_path))) as connection: + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + events = connection.execute( + "SELECT count(*) FROM memory_events WHERE event_type='memory.created'" + ).fetchone() + assert (records[0], events[0]) == (FLOWS, FLOWS) + + +async def test_store_behind_a_held_write_lock_is_retryable_database_in_use(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with closing( + sqlite3.connect(_database(tmp_path), isolation_level=None) + ) as holder: + holder.execute("BEGIN IMMEDIATE") + try: + # Waits out the writer's 5 s SQLite busy timeout. + blocked = await session.preflight_and_store(1) + finally: + holder.execute("ROLLBACK") + assert not blocked["ok"] + assert _is_retryable_busy(blocked), blocked + retried = await session.preflight_and_store(1) + assert retried["ok"], retried + + +async def test_internal_error_is_logged_under_the_returned_correlation_id( + tmp_path, monkeypatch, caplog +): + def explode(*_arguments, **_keywords): + raise RuntimeError("induced-failure-canary") + + # Raised inside the writer transaction, so it reaches the log only through + # the MEMORY_STORE_FAILED exception chain. + monkeypatch.setattr(runtime_services, "_existing_idempotent_event", explode) + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = await session.preflight_and_store(1) + + assert not response["ok"] + error = response["error"] + assert (error["code"], error["retryable"]) == ("INTERNAL_ERROR", False) + assert "induced-failure-canary" not in json.dumps(response) + [record] = [ + record + for record in caplog.records + if record.name == "daem0nmcp.api.v7.responses" + ] + assert error["correlation_id"] in record.getMessage() + assert "induced-failure-canary" in caplog.text + assert "MEMORY_STORE_FAILED" in caplog.text + + +async def test_resource_reads_report_a_full_worker_pool_as_database_in_use(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + pool = BoundedWorkerPool(max_workers=1, thread_name_prefix="test-busy") + release = threading.Event() + occupant = asyncio.create_task(pool.run(lambda: release.wait(timeout=30))) + for _ in range(100): + if pool.in_flight: + break + await asyncio.sleep(0) + try: + repository = SQLiteResourceRepository( + lambda _workspace: pytest.fail("a full pool must not open storage"), + worker_pool=pool, + ) + with pytest.raises(ResourceRepositoryError) as caught: + await repository.read_warnings( + workspace, + ResourceReadRequest("warnings", 10, "updated_at_desc"), + ) + assert caught.value.code == "DATABASE_IN_USE" + finally: + release.set() + await occupant + pool.shutdown() diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index 6900624..5622b20 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -1173,6 +1173,12 @@ async def guidance( self.assertTrue(validated.ok) self.assertIsNone(validated.data.preflight_token) self.assertIsNone(validated.data.expires_at) + # An ok:true draft must say why no token was issued (F-070/F-082). + [warning] = validated.meta.warnings + self.assertEqual("PREFLIGHT_TOKEN_NOT_ISSUED", warning.code) + self.assertIn("record_type (missing)", warning.message) + self.assertIn("content (missing)", warning.message) + self.assertIn("No preflight token issued", warning.message) self.assertEqual(len(observed), 1) self.assertEqual(observed[0][0], "memory_store") self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) @@ -1761,6 +1767,48 @@ def test_record_outcome_uses_a_sanitized_task7_event_receipt(self) -> None: self.assertTrue(validated.data.worked) self.assertFalse(validated.data.idempotent_replay) + def test_transient_writer_contention_is_retryable_database_in_use(self) -> None: + # Pool saturation and SQLite lock timeouts clear on retry (F-020/F-027). + from daem0nmcp.api.v7 import pinned + from daem0nmcp.api.v7.errors import ErrorCode + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError + from daem0nmcp.api.v7.tools import MemoryStoreInput, MemoryStoreOutput + + workspace = Workspace(WORKSPACE_ID, Path.cwd()) + scope = InvocationScope("principal", "session-busy", str(workspace.root)) + gate = _gate() + gate.record_briefing(scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_type": "decision", + "content": "Contention is transient.", + "idempotency_key": "decision-busy-1", + } + token = gate.issue_preflight(scope, "memory_store", target) + handlers = pinned.build_pinned_handlers( + _dependencies( + gate=gate, + scope=scope, + workspace=workspace, + briefing_service=_UnusedService(), + memory_event_writer=_FailingMemoryEventWriter( + RuntimeServiceError("DATABASE_IN_USE") + ), + ) + ) + + response = asyncio.run( + handlers["memory_store"]( + **MemoryStoreInput(**target, preflight_token=token).model_dump() + ) + ) + + validated = MemoryStoreOutput.model_validate(response) + self.assertFalse(validated.ok) + self.assertEqual(ErrorCode.DATABASE_IN_USE, validated.error.code) + self.assertTrue(validated.error.retryable) + self.assertEqual(250, validated.error.retry_after_ms) + def test_record_outcome_maps_runtime_not_found(self) -> None: from daem0nmcp.api.v7 import pinned from daem0nmcp.api.v7.errors import ErrorCode diff --git a/tests/api_v7/test_process_concurrent_writers.py b/tests/api_v7/test_process_concurrent_writers.py new file mode 100644 index 0000000..7010287 --- /dev/null +++ b/tests/api_v7/test_process_concurrent_writers.py @@ -0,0 +1,97 @@ +"""Two stdio server processes writing one workspace store each key exactly once.""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from contextlib import closing +from typing import Any + +from mcp import ClientSession + +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import call, initialize_workspaces, process_client + +SAME_KEY_STORES = 5 +DISTINCT_KEY_STORES = 10 + + +async def _store_until_ok( + session: ClientSession, + workspace_id: str, + key: str, + content: str, +) -> list[dict[str, Any]]: + """Preflight and store, retrying only what the server marks retryable.""" + scope = {"workspace_id": workspace_id} + arguments = {"record_type": "decision", "content": content, "idempotency_key": key} + observed: list[dict[str, Any]] = [] + deadline = asyncio.get_running_loop().time() + 120 + while True: + response = await call( + session, + "memory_preflight", + {**scope, "target_tool": "memory_store", "target_arguments": arguments}, + ) + if response["ok"]: + token = response["data"]["preflight_token"] + response = await call( + session, + "memory_store", + {**scope, **arguments, "preflight_token": token}, + ) + observed.append(response) + if response["ok"]: + return observed + assert response["error"]["code"] == "DATABASE_IN_USE", observed + assert response["error"]["retryable"] is True, observed + assert asyncio.get_running_loop().time() < deadline, observed + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + +async def test_two_stdio_servers_store_each_key_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + scope = {"workspace_id": workspace.workspace_id} + async with ( + process_client(tmp_path, "stdio", log_name="stdio-server-a.log") as first, + process_client(tmp_path, "stdio", log_name="stdio-server-b.log") as second, + ): + sessions = (first, second) + for session in sessions: + assert (await call(session, "session_brief", scope))["ok"] + stores = [ + ("shared-store-key-0001", "One decision stored from two processes") + ] * SAME_KEY_STORES + [ + (f"distinct-store-key-{index:04d}", f"Distinct decision {index}") + for index in range(DISTINCT_KEY_STORES) + ] + results = await asyncio.gather( + *( + _store_until_ok( + sessions[index % 2], workspace.workspace_id, key, content + ) + for index, (key, content) in enumerate(stores) + ) + ) + + responses = [response for observed in results for response in observed] + assert all( + response["ok"] or response["error"]["code"] == "DATABASE_IN_USE" + for response in responses + ), responses + record_ids = {observed[-1]["data"]["record"]["record_id"] for observed in results} + assert len(record_ids) == 1 + DISTINCT_KEY_STORES + replays = [observed[-1]["data"]["idempotent_replay"] for observed in results] + assert replays[:SAME_KEY_STORES].count(False) == 1, replays + + database = resolve_active_database(tmp_path / ".daem0nmcp" / "storage").path + with closing(sqlite3.connect(database)) as connection: + per_stream = dict( + connection.execute( + "SELECT stream_id, count(*) FROM memory_events " + "WHERE event_type='memory.created' GROUP BY stream_id" + ).fetchall() + ) + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + assert per_stream == dict.fromkeys(record_ids, 1) + assert records[0] == 1 + DISTINCT_KEY_STORES diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index 7f31a5f..70718ec 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -332,6 +332,7 @@ async def test_outcome_updates_existing_workspace_record_and_replays( async def test_cancelled_worker_retains_capacity_and_leaves_no_write(self) -> None: """Cancelling an asyncio waiter must not over-admit or commit later.""" + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError from daem0nmcp.bounded_workers import BoundedWorkerBusyError clock_entered = threading.Event() @@ -347,11 +348,13 @@ def blocked_clock() -> datetime: await asyncio.to_thread(clock_entered.wait, 1.0) first.cancel() await asyncio.sleep(0) - with self.assertRaises(BoundedWorkerBusyError): + with self.assertRaises(RuntimeServiceError) as busy: await writer.store( self.workspace, self._store_command(idempotency_key="runtime-store-overflow"), ) + self.assertEqual("DATABASE_IN_USE", busy.exception.code) + self.assertIsInstance(busy.exception.__cause__, BoundedWorkerBusyError) release_clock.set() with self.assertRaises(asyncio.CancelledError): await first From 224e7ee71933e4c9f3cd591ade0bf92109362a7e Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:15:46 -0400 Subject: [PATCH 24/72] fix: let user text mention paths, with one path predicate for everything else Memory content, rationale, tags, context, queries, rule and trigger text, entity names and the server text that quotes them are typed UserText and are exempt from the absolute-path rule in both directions (UD-3, KD-2). Every other string on every WireModel, input and output, keeps the rule through the single is_host_absolute_path predicate, so nothing can be stored and then brick a read. Delete the drifted output guards (_path_safe_success, both _contains_raw_path copies, the resource repository's root scan, and the path checks on content in entity evolution, intelligence summaries, TODO text, capture records and edit descriptions). Event payload checks keep refusing raw path columns and host paths in relative-path keys. context.visibility is validated at the input model. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/code_entity_operations.py | 2 - daem0nmcp/api/v7/discovery_operations.py | 29 ++- daem0nmcp/api/v7/intelligence_operations.py | 8 +- daem0nmcp/api/v7/models.py | 109 +++++++--- daem0nmcp/api/v7/operations.py | 31 +-- daem0nmcp/api/v7/pinned.py | 53 +---- daem0nmcp/api/v7/resource_repository.py | 29 --- daem0nmcp/api/v7/resources.py | 6 +- daem0nmcp/api/v7/runtime_services.py | 35 ---- daem0nmcp/api/v7/tools.py | 217 +++++++++++--------- daem0nmcp/api/v7/utility_operations.py | 4 +- daem0nmcp/capture_candidates.py | 2 - daem0nmcp/edit_bridge.py | 8 +- tests/api_v7/test_models.py | 29 +-- tests/api_v7/test_operations.py | 22 +- tests/api_v7/test_pinned_handlers.py | 21 +- tests/api_v7/test_resource_repository.py | 27 ++- tests/api_v7/test_runtime_services.py | 2 +- 18 files changed, 306 insertions(+), 328 deletions(-) diff --git a/daem0nmcp/api/v7/code_entity_operations.py b/daem0nmcp/api/v7/code_entity_operations.py index 5368379..492082e 100644 --- a/daem0nmcp/api/v7/code_entity_operations.py +++ b/daem0nmcp/api/v7/code_entity_operations.py @@ -49,7 +49,6 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, ) from .public_ids import ( PublicObjectIdNotFound, @@ -939,7 +938,6 @@ def _safe_event_content(state: Mapping[str, Any]) -> str: not isinstance(content, str) or not content or _CONTROL_RE.search(content) is not None - or contains_absolute_filesystem_path(content) ): raise CodeEntityOperationError("CAPABILITY_DEGRADED") return content diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..563e0a0 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -16,7 +16,6 @@ import inspect import json import os -import posixpath import re import secrets import sqlite3 @@ -51,7 +50,13 @@ ) from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import CapabilityState, Page, RecordSummary, RetrievalData +from .models import ( + CapabilityState, + Page, + RecordSummary, + RetrievalData, + _relative_path, +) from .public_ids import ( PublicObjectIdNotFound, PublicObjectIdRepository, @@ -756,21 +761,13 @@ def _partition_metadata( def _safe_code_path(value: object) -> bool: - if ( - not isinstance(value, str) - or not 1 <= len(value) <= 1024 - or "\\" in value - or "\x00" in value - or value.startswith(("/", "~")) - or re.match(r"^[A-Za-z]:", value) is not None - or value in {".", ".."} - ): + if not isinstance(value, str) or not 1 <= len(value) <= 1024 or value == ".": return False - components = value.split("/") - return ( - all(component not in {"", ".", ".."} for component in components) - and posixpath.normpath(value) == value - ) + try: + _relative_path(value) + except ValueError: + return False + return True def _validate_entity_partition( diff --git a/daem0nmcp/api/v7/intelligence_operations.py b/daem0nmcp/api/v7/intelligence_operations.py index c144850..773e496 100644 --- a/daem0nmcp/api/v7/intelligence_operations.py +++ b/daem0nmcp/api/v7/intelligence_operations.py @@ -34,7 +34,6 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, parse_wire_datetime, ) from .public_ids import PublicObjectIdNotFound, PublicObjectIdRepository @@ -369,12 +368,7 @@ def _verified_event(row: sqlite3.Row) -> dict[str, Any]: def _safe_text(value: object) -> str: - if ( - not isinstance(value, str) - or not value - or _CONTROL_RE.search(value) is not None - or contains_absolute_filesystem_path(value) - ): + if not isinstance(value, str) or not value or _CONTROL_RE.search(value) is not None: raise IntelligenceOperationError("CAPABILITY_DEGRADED") return value diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index c440e7e..3eefd62 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -10,7 +10,9 @@ import json import posixpath import re +from collections.abc import Iterator, Mapping from datetime import datetime, timezone +from functools import cache from typing import ( TYPE_CHECKING, Annotated, @@ -19,6 +21,7 @@ Literal, TypeAlias, TypeVar, + get_args, ) from pydantic import ( @@ -55,26 +58,79 @@ _CONTROL_CHAR = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") -def contains_absolute_filesystem_path(value: object) -> bool: - """Return whether a nested public value contains an absolute path.""" +def is_host_absolute_path(value: str) -> bool: + """Return whether text contains an absolute host path or file URI. + + This is the only absolute-path predicate on the v7 wire. ``WireModel`` + applies it to every string except ``UserText`` fields. + """ + + return bool( + _WINDOWS_ABSOLUTE_PATH.search(value) + or _POSIX_ABSOLUTE_PATH.search(value) + or _FILE_URI.search(value) + ) + + +class _UserTextMarker: + """``Annotated`` marker for user-authored or user-derived text.""" + + __slots__ = () + + def __repr__(self) -> str: + return "UserText" + + +_USER_TEXT = _UserTextMarker() + +# Memory content, queries, rule text and server text quoting them may mention +# routes and file paths ("/health", "C:\proj\app.py"). Fields typed with it, +# alone or inside a list, set or dict, are exempt from the absolute-path rule in +# both directions, so anything accepted on input can always be read back. +UserText = Annotated[str, _USER_TEXT] + + +def _carries_user_text(annotation: object) -> bool: + return annotation is _USER_TEXT or any( + _carries_user_text(argument) for argument in get_args(annotation) + ) + + +@cache +def _user_text_fields(model: type[BaseModel]) -> frozenset[str]: + return frozenset( + name + for name, field in model.model_fields.items() + if _USER_TEXT in field.metadata or _carries_user_text(field.annotation) + ) + + +def guarded_strings(value: object, user_text: bool = False) -> Iterator[str]: + """Yield every string, keys included, that the absolute-path rule covers. + + Nested models are walked with their own field annotations, so a model + inside a ``UserText`` container is still checked. + """ - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) - or _POSIX_ABSOLUTE_PATH.search(value) - or _FILE_URI.search(value) - ) - if isinstance(value, dict): - return any( - contains_absolute_filesystem_path(key) - or contains_absolute_filesystem_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(contains_absolute_filesystem_path(item) for item in value) if isinstance(value, BaseModel): - return contains_absolute_filesystem_path(value.__dict__) - return False + exempt = _user_text_fields(type(value)) + for name, item in value.__dict__.items(): + yield from guarded_strings(item, name in exempt) + elif isinstance(value, Mapping): + for key, item in value.items(): + yield from guarded_strings(key, user_text) + yield from guarded_strings(item, user_text) + elif isinstance(value, (list, tuple, set, frozenset)): + for item in value: + yield from guarded_strings(item, user_text) + elif isinstance(value, str) and not user_text: + yield value + + +def contains_absolute_filesystem_path(value: object) -> bool: + """Return whether a nested value has a guarded string with a host path.""" + + return any(is_host_absolute_path(text) for text in guarded_strings(value)) def _relative_path(value: str) -> str: @@ -326,7 +382,8 @@ def _document_validation_error( Field(max_length=MAX_JSON_COLLECTION_ITEMS), ], ) -ContextJsonObject = Annotated[JsonObject, AfterValidator(_context_size)] +UserJsonObject = Annotated[JsonObject, _USER_TEXT] +ContextJsonObject = Annotated[UserJsonObject, AfterValidator(_context_size)] ErrorCodeValue = Annotated[ErrorCode, BeforeValidator(_error_code)] UpperSnakeCode = Annotated[ @@ -348,7 +405,7 @@ def _document_validation_error( ), ] Tag = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=80), AfterValidator(_sanitized), ] @@ -430,7 +487,7 @@ class WireModel(BaseModel): @model_validator(mode="after") def reject_absolute_filesystem_paths(self) -> WireModel: - if contains_absolute_filesystem_path(self.__dict__): + if contains_absolute_filesystem_path(self): raise ValueError("absolute filesystem paths are forbidden on the v7 wire") return self @@ -586,7 +643,7 @@ class RecordSummary(WireModel): record_id: RecordId record_type: RecordType excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), AfterValidator(_sanitized), ] @@ -621,7 +678,7 @@ class EvidenceItem(WireModel): citation: Citation record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), AfterValidator(_sanitized), ] @@ -694,7 +751,7 @@ class RetrievalData(WireModel): items: list[EvidenceItem] = Field(default_factory=list, max_length=50) rendered_context: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] | None @@ -790,6 +847,8 @@ class Preview(WireModel): "EventId", "FactId", "FieldError", + "guarded_strings", + "is_host_absolute_path", "JsonObject", "JsonValue", "MAX_CONTEXT_JSON_BYTES", @@ -822,6 +881,8 @@ class Preview(WireModel): "TokenUsage", "ToolName", "TriggerId", + "UserJsonObject", + "UserText", "UtcDateTime", "VersionId", "WireModel", diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..90ef0f1 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -51,7 +51,7 @@ from ...workspace import Workspace from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import EvidenceRef, Page, RecordSummary +from .models import EvidenceRef, Page, RecordSummary, is_host_absolute_path from .portable_projections import ( ImportFinalizationLease, PortableTransferError, @@ -89,8 +89,9 @@ _MAX_EXPORT_EVENTS = 10_000 _IMPORT_LEASE_RENEW_MARGIN_US = 5 * 60 * 1_000_000 _CURSOR_RE = re.compile(r"^cur_([0-9a-f]{64})$") -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? None: def _reject_raw_paths(value: object) -> None: - if isinstance(value, str): - if ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ): - raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - elif isinstance(value, Mapping): + """Refuse host paths in the structured path fields of an event payload. + + Free text (content, rationale, rule text, context) is user text and may + mention paths, as on the wire. + """ + if isinstance(value, Mapping): for key, item in value.items(): + if key in _RAW_PATH_KEYS and item is not None and item != "": + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") if ( - key in {"file_path", "project_path", "database_path"} - and item is not None - and item != "" + key in _RELATIVE_PATH_KEYS + and isinstance(item, str) + and is_host_absolute_path(item) ): raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - _reject_raw_paths(item) + if key != "context": + _reject_raw_paths(item) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): for item in value: _reject_raw_paths(item) diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 9b612ba..1a873ef 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -4,12 +4,11 @@ import inspect import os -import re from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType -from typing import Any, Protocol, TypeVar +from typing import Any, Protocol from ...covenant import ( ArgumentNormalizationError, @@ -78,38 +77,6 @@ def __init__(self, code: str) -> None: super().__init__(code) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - - -T = TypeVar("T") - - -def _path_safe_success(response: ResponseContext, data: T) -> ApiResponse[T]: - if _contains_raw_path(data): - return response.internal_error() - return response.success(data) - - _EXPECTED_SERVICE_ERRORS = MappingProxyType( { "NOT_FOUND": ( @@ -437,7 +404,7 @@ async def session_brief( data = SessionBriefData.model_validate(assembled) if data.workspace_id != request.workspace_id: raise ValueError("briefing service returned a mismatched workspace") - result = _path_safe_success(response, data) + result = response.success(data) if result.ok: self._dependencies.covenant_gate.record_briefing(scope) return result @@ -538,11 +505,8 @@ async def memory_preflight( else guidance_value ) guidance = PreflightGuidance.model_validate(guidance_result) - if _contains_raw_path(guidance): - return response.internal_error() if not target_is_complete: - return _path_safe_success( - response, + return response.success( PreflightData( guidance=guidance, preflight_token=None, @@ -563,7 +527,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=expires_at, ) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc @@ -744,8 +708,7 @@ async def authorize_linked(selected: Workspace) -> None: if inspect.isawaitable(result_value) else result_value ) - return _path_safe_success( - response, + return response.success( RetrievalData.model_validate(result), ) except Exception as exc: @@ -866,7 +829,7 @@ async def memory_store( stream_version=stored.event.stream_version, idempotent_replay=stored.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -961,7 +924,7 @@ async def memory_record_outcome( worked=recorded.worked, idempotent_replay=recorded.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -1029,7 +992,7 @@ async def system_health( return failure if not request.include_components and data.capability_states: data = data.model_copy(update={"capability_states": []}) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index c1c7a07..30f5ebc 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -336,32 +336,6 @@ def _string_list(value: object) -> list[str]: return decoded -def _workspace_root_tokens(workspace: Workspace) -> tuple[str, ...]: - root = str(workspace.root) - variants = {root.casefold(), root.replace("\\", "/").casefold()} - return tuple(value for value in variants if value) - - -def _reject_canonical_root(value: object, workspace: Workspace) -> None: - tokens = _workspace_root_tokens(workspace) - - def walk(item: object) -> None: - if isinstance(item, str): - normalized = item.casefold() - portable = item.replace("\\", "/").casefold() - if any(token in normalized or token in portable for token in tokens): - raise ValueError("public value contains a canonical workspace root") - elif isinstance(item, Mapping): - for key, child in item.items(): - walk(key) - walk(child) - elif isinstance(item, (list, tuple)): - for child in item: - walk(child) - - walk(value) - - def _validated_now(clock: Callable[[], datetime]) -> datetime: value = clock() if ( @@ -1145,7 +1119,6 @@ def _record_summary( "created_at": _datetime_from_us(row["created_at_us"]), "updated_at": _datetime_from_us(row["updated_at_us"]), } - _reject_canonical_root(public_value, workspace) return RecordSummary.model_validate(public_value), deleted def _read_rules_sync( @@ -1191,7 +1164,6 @@ def _read_rules_sync( "enabled": _flag(row["enabled"]), "created_at": _datetime_from_us(row["created_at_us"]), } - _reject_canonical_root(public_value, workspace) values.append(RuleView.model_validate(public_value)) return values finally: @@ -1417,7 +1389,6 @@ def _read_active_context_sync( "added_at": added_at, "expires_at": expires_at, } - _reject_canonical_root(public_value, workspace) item = ActiveContextItem.model_validate(public_value) ordered_values.append( ( diff --git a/daem0nmcp/api/v7/resources.py b/daem0nmcp/api/v7/resources.py index e19f4c5..b488586 100644 --- a/daem0nmcp/api/v7/resources.py +++ b/daem0nmcp/api/v7/resources.py @@ -23,6 +23,7 @@ AwareDateTime, RecordSummary, RuleId, + UserText, UtcDateTime, WireModel, WorkspaceId, @@ -49,8 +50,9 @@ _UTC_DATETIME_ADAPTER = TypeAdapter(UtcDateTime) _ACCESS_FAILURE = object() +# Rule text is user-authored and may mention paths. BoundedText = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] @@ -91,7 +93,7 @@ class ActiveContextItem(WireModel): priority: Annotated[int, Field(strict=True, ge=-100, le=100)] reason: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index aada0f8..09d08c3 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -11,7 +11,6 @@ import inspect import json import os -import re import sqlite3 import threading import time @@ -89,8 +88,6 @@ SessionBriefInput, ) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) or _POSIX_ABSOLUTE_PATH.search(value) - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - - def _validated_workspace(workspace: Workspace) -> Workspace: if not isinstance(workspace, Workspace): raise RuntimeServiceError("INVALID_WORKSPACE") @@ -1134,8 +1113,6 @@ async def retrieve( raise except Exception: raise RuntimeServiceError("RETRIEVAL_FAILED") from None - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return hydrated async def _retrieve_source( @@ -1254,8 +1231,6 @@ async def retrieve_one( if inspect.isawaitable(value): await value hydrated = compose_federated_results(dict(pairs), query) - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") finally: guard.release() except asyncio.TimeoutError: @@ -1613,8 +1588,6 @@ async def assemble( value = self._reader(workspace, request) if inspect.isawaitable(value): value = await value - if _contains_raw_path(value): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") data = SessionBriefData.model_validate(value) except asyncio.CancelledError: raise @@ -1624,8 +1597,6 @@ async def assemble( raise RuntimeServiceError("BRIEFING_FAILED") from None if data.workspace_id != workspace.workspace_id: raise RuntimeServiceError("BRIEFING_FAILED") - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data @@ -1680,8 +1651,6 @@ async def guidance( raise except Exception: raise RuntimeServiceError("PREFLIGHT_FAILED") from None - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data @@ -1824,8 +1793,6 @@ def __init__( ) except Exception: raise ValueError("health service configuration is invalid") from None - if _contains_raw_path(self._data): - raise ValueError("health service configuration is unsafe") self._workers = BoundedWorkerPool( max_workers=2, thread_name_prefix="daem0nmcp-v7-health", @@ -1904,8 +1871,6 @@ async def inspect( "runtime_diagnostics": diagnostics, } ) - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data diff --git a/daem0nmcp/api/v7/tools.py b/daem0nmcp/api/v7/tools.py index fb5d7bc..3332b86 100644 --- a/daem0nmcp/api/v7/tools.py +++ b/daem0nmcp/api/v7/tools.py @@ -53,6 +53,8 @@ Tag, ToolName, TriggerId, + UserJsonObject, + UserText, UtcDateTime, VersionId, WireModel, @@ -62,10 +64,6 @@ from .registry import PINNED_TOOL_NAMES, ManifestError, ToolSpec from .resources import ActiveContextItem, RuleView -ShortText = Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=80), -] NameText = Annotated[ str, StringConstraints(strict=True, min_length=1, max_length=256), @@ -74,17 +72,34 @@ str, StringConstraints(strict=True, min_length=1, max_length=2000), ] -LongText = Annotated[ - str, +# User-authored or user-derived text, which may mention paths (see UserText). +UserShortText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=80), +] +UserNameText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=256), +] +UserMediumText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=2000), +] +UserLongText = Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] -OptionalMediumText = ( - Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=2000), - ] - | None -) +OptionalUserMediumText = UserMediumText | None +RecordVisibility = Literal["workspace", "private", "shared"] + + +def _record_visibility(context: dict[str, object]) -> dict[str, object]: + if context.get("visibility", "workspace") not in get_args(RecordVisibility): + raise ValueError("context.visibility must be workspace, private or shared") + return context + + +RecordContext = Annotated[ContextJsonObject, AfterValidator(_record_visibility)] IdempotencyKey = Annotated[ str, StringConstraints( @@ -330,8 +345,8 @@ class DiagnosticSummary(WireModel): class TriggerView(WireModel): trigger_id: TriggerId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool updated_at: UtcDateTime @@ -346,7 +361,7 @@ class OutcomeSummary(WireModel): record_id: RecordId worked: bool outcome_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), ] happened_at: UtcDateTime @@ -378,10 +393,10 @@ class SessionBriefData(WireModel): class PreflightGuidance(WireModel): records: list[RecordSummary] = Field(default_factory=list, max_length=20) rules: list[RuleView] = Field(default_factory=list, max_length=20) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class PreflightData(WireModel): @@ -413,7 +428,7 @@ class EditPreflightData(WireModel): class CaptureCandidateView(WireModel): candidate_id: CaptureCandidateId source_kind: Literal["native_edit", "tool_result", "dreaming", "system"] - proposed_record: JsonObject + proposed_record: UserJsonObject provenance: JsonObject created_at: UtcDateTime @@ -521,7 +536,7 @@ class HealthData(WireModel): class TriggerMatch(WireModel): trigger: TriggerView - matched_value: NameText + matched_value: UserNameText records: list[RecordSummary] = Field(default_factory=list, max_length=20) @@ -564,7 +579,7 @@ class HierarchyLayer(WireModel): class CommunitySummary(WireModel): community_id: CommunityId - label: NameText + label: UserNameText level: Annotated[int, Field(ge=0, le=32)] member_count: Annotated[int, Field(ge=0, le=1_000_000)] parent_community_id: CommunityId | None = None @@ -591,7 +606,7 @@ def validate_span(self) -> HighlightSpan: class TextSearchHit(WireModel): record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), ] highlights: list[HighlightSpan] = Field(default_factory=list, max_length=100) @@ -600,15 +615,15 @@ class TextSearchHit(WireModel): class RuleCheckData(WireModel): matched_rules: list[RuleView] = Field(default_factory=list, max_length=50) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class ContextCompressData(WireModel): text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] original_tokens: Annotated[int, Field(ge=0, le=1_000_000)] @@ -625,7 +640,7 @@ class MemoryStoreBatchData(WireModel): class DocumentSource(WireModel): url: HttpsUrl - topic: MediumText + topic: UserMediumText content_hash: ContentHash @@ -637,13 +652,13 @@ class DocumentIngestData(WireModel): class VerifiedClaim(WireModel): - claim: MediumText + claim: UserMediumText status: Literal["supported", "contradicted", "unknown"] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=32) class Contradiction(WireModel): - claim: MediumText + claim: UserMediumText explanation: MediumText evidence_refs: list[EvidenceRef] = Field(min_length=1, max_length=32) @@ -657,8 +672,8 @@ class MemoryVerifyData(WireModel): class SandboxExecutionData(WireModel): success: bool - stdout: Annotated[str, StringConstraints(strict=True, max_length=100_000)] - stderr: Annotated[str, StringConstraints(strict=True, max_length=100_000)] + stdout: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] + stderr: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] exit_status: Annotated[int, Field(ge=-1, le=255)] execution_time_ms: Annotated[int, Field(ge=0, le=60_000)] sanitized_logs: list[MediumText] = Field(default_factory=list, max_length=100) @@ -675,7 +690,7 @@ class CodeIndexData(WireModel): class CodeEntitySummary(WireModel): code_entity_id: CodeEntityId kind: CodeEntityKind - qualified_name: NameText + qualified_name: UserNameText relative_file_path: RelativePath start_line: Annotated[int, Field(ge=1)] end_line: Annotated[int, Field(ge=1)] @@ -703,7 +718,7 @@ class TodoFinding(WireModel): relative_file_path: RelativePath line: Annotated[int, Field(ge=1)] todo_type: TodoType - text: MediumText + text: UserMediumText class CodeTodosStoreData(WireModel): @@ -714,7 +729,7 @@ class CodeTodosStoreData(WireModel): class RefactorProposalData(WireModel): proposal: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] affected_entities: list[CodeEntitySummary] = Field( @@ -729,7 +744,7 @@ class RelationshipView(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] @@ -752,7 +767,7 @@ class MemoryChainTraceData(WireModel): class KnowledgeGraphNode(WireModel): record: RecordSummary - label: NameText + label: UserNameText node_type: Literal["record", "entity", "community"] @@ -769,7 +784,7 @@ class KnowledgeGraphData(WireModel): class KnowledgeGraphRenderData(WireModel): format: Literal["mermaid"] = "mermaid" text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=200) @@ -795,7 +810,7 @@ class CommunityRebuildData(WireModel): class EntitySummary(WireModel): entity_id: EntityId - name: NameText + name: UserNameText entity_type: EntityType mention_count: Annotated[int, Field(ge=0)] manifest_generation: Annotated[int, Field(ge=1)] @@ -810,7 +825,7 @@ class EntityBackfillData(WireModel): class EntityEvolutionItem(WireModel): happened_at: UtcDateTime - summary: MediumText + summary: UserMediumText event_id: EventId @@ -854,7 +869,7 @@ class ExportEvent(WireModel): event_type: NameText happened_at: UtcDateTime content_hash: ContentHash - payload: JsonObject + payload: UserJsonObject TransferSessionId = Annotated[ @@ -872,7 +887,7 @@ class PortableVectorPoint(WireModel): point_id: Annotated[ str, StringConstraints(strict=True, min_length=1, max_length=256) ] - payload: JsonObject + payload: UserJsonObject vector: list[Annotated[float, Field(allow_inf_nan=False)]] = Field( min_length=1, max_length=4096 ) @@ -898,7 +913,7 @@ class ExportBundle(WireModel): page_proof: list[ContentHash] = Field(default_factory=list, max_length=32) next_cursor: Cursor | None = None complete: bool = True - legacy_rows: list[JsonObject] = Field(default_factory=list, max_length=4096) + legacy_rows: list[UserJsonObject] = Field(default_factory=list, max_length=4096) vector_points: list[PortableVectorPoint] = Field( default_factory=list, max_length=4096 ) @@ -970,7 +985,7 @@ class WorkspaceLinkView(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship - label: OptionalMediumText = None + label: OptionalUserMediumText = None class WorkspaceConsolidateData(WireModel): @@ -1016,14 +1031,14 @@ class RuleEvolutionData(WireModel): class DebateRound(WireModel): round_number: Annotated[int, Field(ge=1, le=20)] - advocate: MediumText - challenger: MediumText + advocate: UserMediumText + challenger: UserMediumText class DecisionDebateData(WireModel): rounds: list[DebateRound] = Field(min_length=1, max_length=20) synthesis: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] consensus_record_id: RecordId @@ -1033,7 +1048,7 @@ class DecisionDebateData(WireModel): class SessionBriefInput(WireModel): workspace_id: WorkspaceId - focus_areas: list[ShortText] = Field(default_factory=list, max_length=10) + focus_areas: list[UserShortText] = Field(default_factory=list, max_length=10) warning_limit: Annotated[int, Field(ge=0, le=50)] = 10 failure_limit: Annotated[int, Field(ge=0, le=50)] = 10 @@ -1041,10 +1056,10 @@ class SessionBriefInput(WireModel): class MemoryPreflightInput(WireModel): workspace_id: WorkspaceId target_tool: ProtectedToolName - target_arguments: JsonObject + target_arguments: UserJsonObject description: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=1000), ] | None @@ -1054,7 +1069,7 @@ class MemoryPreflightInput(WireModel): class MemoryRecallInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] limit: Annotated[int, Field(ge=1, le=50)] = 10 @@ -1080,19 +1095,19 @@ def validate_candidate_limit(self) -> MemoryRecallInput: class MemoryStoreInput(WireModel): workspace_id: WorkspaceId record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1106,7 +1121,7 @@ def validate_procedure_steps(self) -> MemoryStoreInput: class EditPreflightInput(WireModel): workspace_id: WorkspaceId edit_request_id: EditRequestId - description: MediumText + description: UserMediumText class MemoryCaptureListInput(WireModel): @@ -1120,11 +1135,11 @@ class MemoryCapturePromoteInput(WireModel): candidate_id: CaptureCandidateId record_type: RecordType content: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8_000), ] - rationale: OptionalMediumText = None - context: ContextJsonObject = Field(default_factory=dict) + rationale: OptionalUserMediumText = None + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=16) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1134,7 +1149,7 @@ class MemoryRecordOutcomeInput(WireModel): workspace_id: WorkspaceId record_id: RecordId outcome_text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] worked: bool @@ -1157,7 +1172,7 @@ class ContextTriggersMatchInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath | None = None tags: list[Tag] = Field(default_factory=list, max_length=32) - entities: list[NameText] = Field(default_factory=list, max_length=32) + entities: list[UserNameText] = Field(default_factory=list, max_length=32) limit: Annotated[int, Field(ge=1, le=20)] = 5 @@ -1182,7 +1197,7 @@ class MemoryRecallFileInput(WireModel): class MemoryRecallEntityInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None cursor: Cursor | None = None limit: Annotated[int, Field(ge=1, le=100)] = 20 @@ -1197,7 +1212,7 @@ def validate_selector(self) -> MemoryRecallEntityInput: class MemoryRecallHierarchicalInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] include_members: bool = False @@ -1207,7 +1222,7 @@ class MemoryRecallHierarchicalInput(WireModel): class MemorySearchTextInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1218,36 +1233,36 @@ class MemorySearchTextInput(WireModel): class RuleCheckInput(WireModel): workspace_id: WorkspaceId - proposed_action: MediumText + proposed_action: UserMediumText context: ContextJsonObject = Field(default_factory=dict) class ContextCompressInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] rate: Annotated[float, Field(ge=0.1, le=1, allow_inf_nan=False)] | None = None - content_type: ShortText | None = None + content_type: UserShortText | None = None preserve_code: bool = True class MemoryCreate(WireModel): record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) @model_validator(mode="after") def validate_procedure_steps(self) -> MemoryCreate: @@ -1268,7 +1283,7 @@ class MemoryLinkInput(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] = 1.0 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1296,7 +1311,7 @@ class MemoryPinSetInput(WireModel): class ActiveContextAddInput(WireModel): workspace_id: WorkspaceId record_id: RecordId - reason: OptionalMediumText = None + reason: OptionalUserMediumText = None priority: Annotated[int, Field(ge=-100, le=100)] = 0 expires_at: AwareDateTime | None = None preflight_token: PreflightToken @@ -1317,7 +1332,7 @@ class ActiveContextClearInput(WireModel): class DocumentIngestUrlInput(WireModel): workspace_id: WorkspaceId url: HttpsUrl - topic: MediumText + topic: UserMediumText chunk_size: Annotated[int, Field(ge=256, le=16_000)] = 2000 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1326,7 +1341,7 @@ class DocumentIngestUrlInput(WireModel): class MemoryVerifyInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] categories: RecordTypeSet | None = None @@ -1337,7 +1352,7 @@ class MemoryVerifyInput(WireModel): class SandboxExecutePythonInput(WireModel): workspace_id: WorkspaceId code: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] timeout_seconds: Annotated[int, Field(ge=1, le=60)] = 30 @@ -1356,7 +1371,7 @@ class CodeIndexInput(WireModel): class CodeSearchInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1367,7 +1382,7 @@ class CodeSearchInput(WireModel): class CodeImpactAnalyzeInput(WireModel): workspace_id: WorkspaceId code_entity_id: CodeEntityId | None = None - qualified_name: NameText | None = None + qualified_name: UserNameText | None = None max_depth: Annotated[int, Field(ge=1, le=10)] = 3 @model_validator(mode="after") @@ -1401,27 +1416,27 @@ class CodeTodosScanAndStoreInput(WireModel): class CodeRefactorProposeInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath - objective: OptionalMediumText = None + objective: OptionalUserMediumText = None class RuleCreateInput(WireModel): workspace_id: WorkspaceId - trigger: MediumText - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + trigger: UserMediumText + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] = 0 idempotency_key: IdempotencyKey preflight_token: PreflightToken class RulePatch(WireModel): - trigger: MediumText | None = None - must_do: list[MediumText] | None = Field(default=None, max_length=50) - must_not: list[MediumText] | None = Field(default=None, max_length=50) - ask_first: list[MediumText] | None = Field(default=None, max_length=50) - warnings: list[MediumText] | None = Field(default=None, max_length=50) + trigger: UserMediumText | None = None + must_do: list[UserMediumText] | None = Field(default=None, max_length=50) + must_not: list[UserMediumText] | None = Field(default=None, max_length=50) + ask_first: list[UserMediumText] | None = Field(default=None, max_length=50) + warnings: list[UserMediumText] | None = Field(default=None, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] | None = None enabled: bool | None = None @@ -1449,8 +1464,8 @@ class RuleListInput(WireModel): class ContextTriggerCreateInput(WireModel): workspace_id: WorkspaceId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool = True idempotency_key: IdempotencyKey @@ -1490,7 +1505,7 @@ class KnowledgeGraphGetInput(WireModel): record_ids: RecordIdSet500 | None = None query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1548,7 +1563,7 @@ class EntityBackfillInput(WireModel): class EntityEvolutionTraceInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None include_invalidated: bool = False @@ -1606,13 +1621,13 @@ class MemoryDuplicatesCleanupInput(MemoryDuplicatesPreviewInput): class MemoryCompactionPreviewInput(WireModel): workspace_id: WorkspaceId summary: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] limit: Annotated[int, Field(ge=1, le=100)] = 10 query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1674,7 +1689,7 @@ class WorkspaceLinkInput(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship = "related" - label: OptionalMediumText = None + label: OptionalUserMediumText = None preflight_token: PreflightToken @model_validator(mode="after") @@ -1740,13 +1755,13 @@ class RuleEvolutionAnalyzeInput(WireModel): class DecisionDebateInput(WireModel): workspace_id: WorkspaceId - topic: MediumText + topic: UserMediumText advocate_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] challenger_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] max_rounds: Annotated[int, Field(ge=1, le=20)] = 5 diff --git a/daem0nmcp/api/v7/utility_operations.py b/daem0nmcp/api/v7/utility_operations.py index db5deb2..2d07335 100644 --- a/daem0nmcp/api/v7/utility_operations.py +++ b/daem0nmcp/api/v7/utility_operations.py @@ -22,7 +22,7 @@ from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import Page, contains_absolute_filesystem_path +from .models import Page from .runtime_protocols import WorkerPool from .tasks import await_task_terminal from .tools import ContextCompressData, RefactorProposalData, TodoFinding @@ -278,7 +278,7 @@ def _findings( if todo_type not in selected_types: continue rendered = line.strip()[:2000] - if not rendered or contains_absolute_filesystem_path(rendered): + if not rendered: continue try: findings.append( diff --git a/daem0nmcp/capture_candidates.py b/daem0nmcp/capture_candidates.py index 6856e17..9c56d43 100644 --- a/daem0nmcp/capture_candidates.py +++ b/daem0nmcp/capture_candidates.py @@ -151,8 +151,6 @@ def _validated_record(value: Mapping[str, Any]) -> dict[str, Any]: raise ValueError("capture tags are invalid") if len(tags) != len(set(tags)): raise ValueError("capture tags must be unique") - if contains_absolute_filesystem_path(record): - raise ValueError("capture record contains a public absolute path") return { "record_type": record["record_type"], "content": content, diff --git a/daem0nmcp/edit_bridge.py b/daem0nmcp/edit_bridge.py index 4c02fdd..79f1445 100644 --- a/daem0nmcp/edit_bridge.py +++ b/daem0nmcp/edit_bridge.py @@ -558,11 +558,9 @@ def _issue_receipt_sync( description: str, ) -> EditReceipt: now_us = _datetime_us(self.clock()) - if ( - not isinstance(description, str) - or not 1 <= len(description) <= 2_000 - or contains_absolute_filesystem_path(description) - ): + # The description is user text (it may mention paths) and only its + # hash is stored. + if not isinstance(description, str) or not 1 <= len(description) <= 2_000: raise EditBridgeError("INVALID_ARGUMENT", "edit description rejected") if os.path.normcase(str(workspace.root.resolve())) != scope.canonical_workspace: raise EditBridgeError("TOKEN_SCOPE_MISMATCH", "workspace scope changed") diff --git a/tests/api_v7/test_models.py b/tests/api_v7/test_models.py index 17b4a9d..d461679 100644 --- a/tests/api_v7/test_models.py +++ b/tests/api_v7/test_models.py @@ -247,9 +247,7 @@ def test_json_values_are_strict_finite_bounded_and_duplicate_safe(self) -> None: '{"tool":"memory_store","arguments":{"x":NaN}}' ) - def test_every_wire_model_rejects_absolute_paths_in_free_text_and_json( - self, - ) -> None: + def test_wire_models_reject_absolute_paths_outside_user_text(self) -> None: _, models = _load(self) with self.assertRaises(ValidationError): @@ -262,18 +260,21 @@ def test_every_wire_model_rejects_absolute_paths_in_free_text_and_json( }, ) + summary = { + "record_id": RECORD_ID, + "record_type": "warning", + "excerpt": "Do not read /home/private/.ssh/id_rsa", + "tags": ["C:\\proj"], + "relative_file_path": None, + "current_status": "current", + "content_hash": "a" * 64, + "created_at": NOW, + "updated_at": NOW, + } + # Excerpts and tags are user text (UD-3); the path field is not. + models.RecordSummary(**summary) with self.assertRaises(ValidationError): - models.RecordSummary( - record_id=RECORD_ID, - record_type="warning", - excerpt="Do not read /home/private/.ssh/id_rsa", - tags=[], - relative_file_path=None, - current_status="current", - content_hash="a" * 64, - created_at=NOW, - updated_at=NOW, - ) + models.RecordSummary(**{**summary, "relative_file_path": "/home/x"}) class EnvelopeTests(unittest.TestCase): diff --git a/tests/api_v7/test_operations.py b/tests/api_v7/test_operations.py index b042914..becdf49 100644 --- a/tests/api_v7/test_operations.py +++ b/tests/api_v7/test_operations.py @@ -755,23 +755,25 @@ def active_connection(*_args, **_kwargs): self.assertEqual(10_001, bundle.manifest["event_count"]) snapshot_builder.assert_called_once() - async def test_export_rejects_nested_absolute_path_strings(self) -> None: - secret = str(self.fixture.root / "private" / "nested-secret.txt") + async def test_export_carries_user_text_that_mentions_paths(self) -> None: + # Context is user text (UD-3); only the raw path columns are refused. + note = "Crash fixed in C:\\proj\\src\\app.py; see /health" self.fixture.append( "nested path source", occurred_at_us=1_700_000_000_000_000, recorded_at_us=1_700_000_000_000_100, - context_note=secret, + context_note=note, ) - with self.assertRaises(Exception) as caught: - await self._operations()["workspace_export"]( - workspace=self.fixture.workspace, - request=_request("workspace_export", workspace_id=WORKSPACE_ID), - ) + bundle = await self._operations()["workspace_export"]( + workspace=self.fixture.workspace, + request=_request("workspace_export", workspace_id=WORKSPACE_ID), + ) - self.assertEqual("WORKSPACE_PATH_ESCAPE", caught.exception.code) - self.assertNotIn(secret, str(caught.exception)) + self.assertEqual( + {"note": note}, + bundle.events[0].payload["data"]["record"]["context"], + ) async def test_cancelled_sqlite_work_propagates_and_finishes_privately( self, diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index 6900624..e79a5e7 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -110,12 +110,23 @@ async def guidance(self, *args: object, **kwargs: object) -> object: class _LeakyPreflightService: async def guidance(self, *args: object, **kwargs: object) -> object: - from daem0nmcp.api.v7.tools import PreflightGuidance - del args, kwargs - return PreflightGuidance( - warnings=[r"Inspect D:\private\workspace\policy.txt"], - ) + # Rule text may mention paths (UD-3); a path field may not. + return { + "warnings": [r"Inspect D:\private\workspace\notes.txt"], + "records": [ + { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": "Guarded path field.", + "relative_file_path": r"D:\private\workspace\policy.txt", + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + ], + } class _ExplodingRecallService: diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index 45a43a4..b97fab7 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -469,22 +469,21 @@ async def test_memory_flags_are_honored_without_losing_row_state(self) -> None: self.assertEqual(by_id[deleted].item.current_status, "invalidated") self.assertTrue(by_id[deleted].deleted) - async def test_canonical_root_embedded_in_public_text_fails_closed(self) -> None: - # Catches free-text fields bypassing the path-safe structured field policy. - self.fixture.add_record( - 12, - content=f"do not expose {self.fixture.workspace_root / 'secret.py'}", - ) + async def test_user_text_naming_the_workspace_root_reads_back(self) -> None: + # User text may mention paths, the root included (UD-3); one such + # warning must not make the resource unreadable. + content = f"see {self.fixture.workspace_root / 'app.py'}" + record_id = self.fixture.add_record(12, content=content) repository = self._repository() - from daem0nmcp.api.v7.resource_repository import ResourceRepositoryError - with self.assertRaises(ResourceRepositoryError): - await repository.read_warnings( - self.fixture.workspace, - ResourceReadRequest( - kind="warnings", limit=1, order_by="updated_at_desc" - ), - ) + rows = await repository.read_warnings( + self.fixture.workspace, + ResourceReadRequest(kind="warnings", limit=1, order_by="updated_at_desc"), + ) + + self.assertEqual( + [(record_id, content)], [(r.item.record_id, r.item.excerpt) for r in rows] + ) async def test_rules_use_canonical_ids_and_highest_priority_enabled_order( self, diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index 7f31a5f..ffffc8d 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -1239,7 +1239,7 @@ async def leaky_reader(_workspace, _request): with self.assertRaisesRegex( RuntimeServiceError, - "UNSAFE_SERVICE_OUTPUT", + "BRIEFING_FAILED", ) as leak: await BasicBriefingService(reader=leaky_reader).assemble( self.workspace, From e6a087e8957b76e7c4b56c28b3828f008f9378cb Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:15:51 -0400 Subject: [PATCH 25/72] test: prove path-bearing user text round-trips through every tool test_path_policy stores the UD-3 examples, reads back a pre-fix record, seeds every free-text input field found from the tool models plus migrated-style memory, rule, trigger, entity and community rows with paths, then calls every read tool and resource. The isolation sweep now stores a note naming each target's root and scans for host paths only outside UserText fields, located through the response models. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_path_policy.py | 619 ++++++++++++++++++ .../test_process_workspace_isolation_sweep.py | 104 ++- 2 files changed, 707 insertions(+), 16 deletions(-) create mode 100644 tests/api_v7/test_path_policy.py diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py new file mode 100644 index 0000000..f7df0f3 --- /dev/null +++ b/tests/api_v7/test_path_policy.py @@ -0,0 +1,619 @@ +"""Path policy on the v7 wire (KD-2, UD-3). + +User text (memory content, rule text, queries, and server text quoting them) +may mention routes and file paths. Every other string keeps the +absolute-path rule, input and output, through one predicate. Anything the +server accepts must read back: nothing can be stored and then brick a read. +""" + +from __future__ import annotations + +import copy +import hashlib +import sqlite3 +import time +import types +from contextlib import closing +from pathlib import Path +from typing import Annotated, Any, Literal, Union, get_args, get_origin, get_type_hints + +import pytest +from fastmcp import Client +from pydantic import BaseModel, ValidationError, create_model + +from daem0nmcp.api.v7.models import ( + Cursor, + JsonObject, + RelativePath, + WireModel, + _user_text_fields, +) +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.tools import ( + TOOL_DATA_MODELS, + TOOL_INPUT_MODELS, + DiagnosticSummary, + HealthData, + HttpsUrl, + MediumText, + MemoryStoreInput, +) +from daem0nmcp.config import Settings +from daem0nmcp.discovery_projection import ( + CommunityProjectionSeed, + DiscoveryProjectionBuilder, + EntityProjectionSeed, + EntityRecordSeed, +) +from daem0nmcp.event_store import ( + EventCommand, + EventStore, + GovernanceEventCommand, + GovernanceEventStore, +) +from daem0nmcp.retrieval.specialized_projection import SpecializedProjectionBuilder +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces +from tests.api_v7.test_process_workspace_isolation_sweep import ( + CANARY, + CROSS_WORKSPACE_TOOLS, + GATE_CODES, + PROFILE_ENVIRONMENT, + _code, + _is_protected, + _mint_tokens, + _seed_workspace_a, + _sweep_arguments, + _target_call, +) + +# The UD-3 examples: a regex escape, routes, and a Windows file location. +PATH_CONTENT = ("\\\\d+", "/health", "GET /api/v1/users", "C:\\proj\\src\\app.py") +# Mechanical-coverage seeds (R-26). +SEEDS = ("C:\\proj\\x.py", "/health", "\\d+") +NEUTRAL = "plain neutral words" +# A path in text is the user's business; these outcomes mean it was refused. +REFUSALS = frozenset( + {"INVALID_ARGUMENT", "INTERNAL_ERROR", "WORKSPACE_PATH_ESCAPE", "SCHEMA_REJECTED"} +) +RESOURCE_KINDS = ("warnings", "failures", "rules", "active-context") +TOKEN = "preflight-token-placeholder-0001" + + +def _invoker(client: Client): + """Call a tool; a schema rejection by FastMCP becomes SCHEMA_REJECTED.""" + + async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await client.call_tool(tool, arguments, raise_on_error=False) + if result.structured_content is None: + assert result.is_error, tool + return {"ok": False, "error": {"code": "SCHEMA_REJECTED"}} + return result.structured_content + + return invoke + + +# --------------------------------------------------------------------------- +# Model-level checks. + + +def _store_input(**overrides: Any) -> dict[str, Any]: + return { + "workspace_id": "ws_" + "0" * 24, + "record_type": "decision", + "content": "text", + "idempotency_key": "path-policy-0001", + "preflight_token": TOKEN, + **overrides, + } + + +@pytest.mark.parametrize("path", ["C:\\x", "/etc/x"]) +def test_relative_file_path_rejects_host_paths(path): + with pytest.raises(ValidationError): + MemoryStoreInput.model_validate(_store_input(relative_file_path=path)) + + +@pytest.mark.parametrize("text", [*PATH_CONTENT, *SEEDS]) +def test_user_text_input_fields_accept_paths(text): + validated = MemoryStoreInput.model_validate( + _store_input( + content=text, + rationale=text, + tags=[text], + context={"note": text, text: [text]}, + ) + ) + assert validated.content == text + + +def test_server_generated_fields_reject_host_paths(): + with pytest.raises(ValidationError): + DiagnosticSummary(code="CODE_INDEX_SKIPPED", message="C:\\x") + health = { + "package_version": "7.0.0", + "protocol_version": "2025-06-18", + "supported_transports": {"stdio"}, + "auth_mode": "loopback", + "task_support": {"name": "tasks", "status": "ready"}, + } + HealthData.model_validate(health) + with pytest.raises(ValidationError): + HealthData.model_validate( + { + **health, + "capability_states": [ + { + "name": "storage", + "status": "failed", + "remediation": "Repair C:\\x", + } + ], + } + ) + + +def _models(root: type[BaseModel], seen: set[type[BaseModel]]) -> None: + if root in seen: + return + seen.add(root) + for field in root.model_fields.values(): + for model in _nested_models(field.annotation): + _models(model, seen) + + +def _nested_models(annotation: object): + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + yield annotation + for argument in get_args(annotation): + yield from _nested_models(argument) + + +def _sample(annotation: object, text: str) -> object: + """A value of ``annotation``'s shape whose every string is ``text``.""" + origin = get_origin(annotation) + if origin is Annotated: + return _sample(get_args(annotation)[0], text) + if origin in (Union, types.UnionType): + options = [item for item in get_args(annotation) if item is not type(None)] + return _sample(options[0], text) + if origin in (list, set, frozenset): + return [_sample(get_args(annotation)[0], text)] + if origin is dict: + return {text: _sample(get_args(annotation)[1], text)} + if annotation is JsonObject: + return {"note": text, text: [text]} + assert annotation is str, annotation + return text + + +def test_user_text_round_trips_into_every_output_field_that_carries_it(): + """Each string a UserText input accepts validates in every UserText output. + + Output fields are found from the models, so a new UserText field is + covered without editing this test. Each probe inherits the model-wide + path check from ``WireModel``. + """ + outputs: set[type[BaseModel]] = set() + for model in ( + *TOOL_DATA_MODELS.values(), + WarningResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + ActiveContextResourceDocument, + ): + _models(model, outputs) + carriers = { + (model.__name__, name): Annotated[ + (field.annotation, *field.metadata) # type: ignore[valid-type] + ] + if field.metadata + else field.annotation + for model in outputs + for name, field in model.model_fields.items() + if name in _user_text_fields(model) + } + assert len(carriers) > 30, sorted(carriers) + for text in (*PATH_CONTENT, *SEEDS): + MemoryStoreInput.model_validate(_store_input(content=text)) + for (model_name, name), annotation in carriers.items(): + probe = create_model( + f"{model_name}_{name}", __base__=WireModel, value=(annotation, ...) + ) + probe.model_validate({"value": _sample(annotation, text)}) + # The probe is meaningful: a guarded text field refuses the same text. + guarded = create_model("Guarded", __base__=WireModel, value=(MediumText, ...)) + if text != "\\\\d+" and text != "\\d+": + with pytest.raises(ValidationError): + guarded.model_validate({"value": text}) + + +# --------------------------------------------------------------------------- +# Rows as a migrated or pre-fix database carries them. + + +def _hex(*parts: object) -> str: + return hashlib.sha256(repr(parts).encode()).hexdigest() + + +def _now_us() -> int: + return time.time_ns() // 1_000 + + +def _memory_record(record_type: str, text: str) -> dict[str, Any]: + return { + "record_type": record_type, + "legacy_type": None, + "content": f"Migrated note: {text}", + "rationale": text, + "context": {"note": text}, + "tags": [text], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "migration", + "source_model": None, + "deleted_at_us": None, + } + + +def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[str]: + """Write path-bearing records, a rule, a trigger, entities and a community.""" + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + record_ids: list[str] = [] + with closing(sqlite3.connect(database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + now = _now_us() - 1_000_000 + for index, text in enumerate(texts): + for record_type in ("decision", "warning"): + record_id = "mem_" + _hex("row", text, record_type) + EventStore(connection).append_and_project( + EventCommand( + workspace_id=workspace_id, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=now + index, + recorded_at_us=now + index, + actor_type="migration", + payload={"record": _memory_record(record_type, text)}, + ) + ) + record_ids.append(record_id) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="rule_" + _hex("rule", text), + stream_kind="rule", + event_type="rule.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "rule_id": "rule_" + _hex("rule", text), + "trigger": text, + "must_do": [text], + "must_not": [text], + "ask_first": [text], + "warnings": [text], + "priority": 0, + "enabled": True, + "created_at_us": now, + "updated_at_us": now, + }, + ) + ) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="trg_" + _hex("trigger", text), + stream_kind="trigger", + event_type="context_trigger.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "trigger_id": "trg_" + _hex("trigger", text), + "trigger_type": "tag", + "pattern": text, + "recall_query": text, + "categories": [], + "enabled": True, + "priority": 0, + "created_at_us": now, + "updated_at_us": now, + "deleted_at_us": None, + }, + ) + ) + connection.commit() + SpecializedProjectionBuilder(connection).rebuild(workspace_id, "graph") + DiscoveryProjectionBuilder(connection).populate_graph( + workspace_id, + entities=tuple( + EntityProjectionSeed( + name=text, + entity_type="file", + records=(EntityRecordSeed(record_ids[2 * index]),), + ) + for index, text in enumerate(texts) + ), + communities=( + CommunityProjectionSeed( + source_key="paths", + label=texts[0], + level=0, + member_record_ids=tuple(record_ids), + ), + ), + ) + connection.commit() + return record_ids + + +def _server(roots: tuple[Path, ...]): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(roots[0]), + workspace_roots=[str(root) for root in roots], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, + ) + + +async def test_path_content_is_stored_and_read_back(tmp_path): + """UD-3 content stores once and every read of it succeeds, pre-fix rows too.""" + root = tmp_path / "project" + (workspace,) = await initialize_workspaces((root,)) + ws = workspace.workspace_id + poisoned = _insert_rows(root, ws, ("\\\\d+",)) + async with Client(_server((root,))) as client: + invoke = _invoker(client) + + async def ok(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await _target_call(invoke, ws, tool, arguments) + assert result["ok"], (tool, arguments, result["error"]) + return result["data"] + + brief = await ok("session_brief", {}) + assert poisoned[1] in {item["record_id"] for item in brief["warnings"]} + for index, text in enumerate(PATH_CONTENT): + for record_type in ("decision", "warning"): + stored = await ok( + "memory_store", + { + "record_type": record_type, + "content": f"Remember {text}", + "idempotency_key": f"path-content-{index}-{record_type}", + }, + ) + record_id = stored["record"]["record_id"] + assert stored["record"]["excerpt"] == f"Remember {text}" + brief = await ok("session_brief", {}) + if record_type == "warning": + assert record_id in { + item["record_id"] for item in brief["warnings"] + } + await ok("memory_recall", {"query": text}) + await ok("memory_search_text", {"query": text}) + preflight = await invoke( + "memory_preflight", + { + "workspace_id": ws, + "target_tool": "memory_store", + "target_arguments": { + "record_type": record_type, + "content": text, + "idempotency_key": f"path-preflight-{index}", + }, + "description": text, + }, + ) + assert preflight["ok"], preflight["error"] + await ok( + "memory_record_outcome", + { + "record_id": record_id, + "outcome_text": text, + "worked": False, + "idempotency_key": f"path-outcome-{index}-{record_type}", + }, + ) + await ok("session_brief", {}) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{ws}/{kind}") + + +# --------------------------------------------------------------------------- +# Mechanical coverage (R-26). + + +def _kind(annotation: object) -> object: + """``"str"``, ``"json"``, a model class, ``("list", kind)`` or ``None``.""" + if annotation in (Cursor, RelativePath, HttpsUrl): + return None + origin = get_origin(annotation) + if origin in (Union, types.UnionType): + kinds = {_kind(item) for item in get_args(annotation) if item is not type(None)} + return kinds.pop() if len(kinds) == 1 else None + if origin is Annotated: + base, *metadata = get_args(annotation) + if any(getattr(item, "pattern", None) for item in metadata): + return None + return _kind(base) + if origin in (list, set, frozenset): + inner = _kind(get_args(annotation)[0]) + return None if inner is None else ("list", inner) + if origin is Literal: + return None + if annotation is str: + return "str" + if annotation is JsonObject: + return "json" + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation + return None + + +def _free_text_paths(model: type[BaseModel], base: dict[str, Any], prefix=()): + """Every free-text-typed field of ``model``, read from its annotations. + + Nested models are followed where ``base`` supplies them. + """ + hints = get_type_hints(model, include_extras=True) + for name in model.model_fields: + kind = _kind(hints[name]) + if kind is None: + continue + if kind in ("str", "json") or kind == ("list", "str"): + yield (*prefix, name), kind + elif isinstance(kind, type) and isinstance(base.get(name), dict): + yield from _free_text_paths(kind, base[name], (*prefix, name)) + elif ( + isinstance(kind, tuple) + and isinstance(kind[1], type) + and isinstance(base.get(name), list) + and base[name] + ): + yield from _free_text_paths(kind[1], base[name][0], (*prefix, name, 0)) + + +def _seeded(base: dict[str, Any], path: tuple, kind: object, text: str) -> dict: + arguments = copy.deepcopy(base) + target = arguments + for step in path[:-1]: + target = target[step] + value: object = {"json": {"note": text, text: [text]}, "str": text}.get( + kind, + [text], # type: ignore[arg-type] + ) + target[path[-1]] = value + return arguments + + +def _fresh_keys(arguments: dict[str, Any], label: str) -> dict[str, Any]: + if "idempotency_key" in arguments: + digest = hashlib.sha256(label.encode()).hexdigest()[:40] + return {**arguments, "idempotency_key": f"mech-{digest}"} + return arguments + + +def _own_read_arguments(arguments: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in arguments.items() + if key not in {"cursor", "after_cursor"} + } + + +async def _seed_free_text(invoke, workspace_id: str, tool: str, arguments: dict): + """Call ``tool`` with each free-text field set to each seed. + + A path must not change the outcome into a refusal: each seeded call + either matches the same call with neutral text or is not a refusal. + """ + base = _own_read_arguments(arguments) + labels = [] + for path, kind in _free_text_paths(TOOL_INPUT_MODELS[tool], base): + label = f"{tool}.{'.'.join(map(str, path))}" + labels.append(label) + neutral = await _target_call( + invoke, + workspace_id, + tool, + _fresh_keys(_seeded(base, path, kind, NEUTRAL), label + NEUTRAL), + ) + for text in SEEDS: + response = await _target_call( + invoke, + workspace_id, + tool, + _fresh_keys(_seeded(base, path, kind, text), label + text), + ) + code = _code(response) + assert code == _code(neutral) or code not in REFUSALS, ( + label, + text, + response["error"], + _code(neutral), + ) + return labels + + +async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_path): + """Seed every free-text input field and DB row with paths; read everything. + + An input field typed as free text but not ``UserText`` refuses the seed + (``INVALID_ARGUMENT``), and an output field carrying it fails the read + (``INTERNAL_ERROR``); either fails this test. + """ + a_root, b_root = tmp_path / "alpha", tmp_path / "beta" + alpha, beta = await initialize_workspaces((a_root, b_root)) + a, b = alpha.workspace_id, beta.workspace_id + (a_root / "src").mkdir() + (a_root / "src" / "owned.py").write_text( + f"def {CANARY}_function():\n return 1 # TODO: see C:\\proj\\x.py /health\n", + encoding="utf-8", + ) + seeded: list[str] = [] + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + seed = await _seed_workspace_a(invoke, a, b) + await _mint_tokens(invoke, a, b, seed) + writes = { + **_sweep_arguments(seed), + "workspace_link": {"linked_workspace_id": b}, + } + for tool, arguments in sorted(writes.items()): + seeded += await _seed_free_text(invoke, a, tool, arguments) + + # Rows as a migrated v6 database or a pre-fix write left them. + _insert_rows(a_root, a, SEEDS) + + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + await _mint_tokens(invoke, a, b, seed) + entities = await invoke("entity_list", {"workspace_id": a, "limit": 100}) + assert set(SEEDS) <= {item["name"] for item in entities["data"]["items"]} + seed.ids["entity"] = entities["data"]["items"][0]["entity_id"] + communities = await invoke("community_list", {"workspace_id": a}) + if communities["ok"] and communities["data"]["items"]: + seed.ids["community"] = communities["data"]["items"][0]["community_id"] + failures: dict[str, str] = {} + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if _is_protected(tool) or tool in CROSS_WORKSPACE_TOOLS: + continue + plain = _fresh_keys(_own_read_arguments(arguments), tool) + response = await invoke(tool, {"workspace_id": a, **plain}) + if _code(response) != "ok" and _code(response) not in GATE_CODES: + failures[tool] = _code(response) + # Query-like fields searching for the paths. + await _seed_free_text(invoke, a, tool, arguments) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{a}/{kind}") + # edit_preflight needs a native edit request, which only the edit hook makes. + assert failures == {"edit_preflight": "NOT_FOUND"}, failures + # Nested and cross-workspace inputs are reached too. + assert { + "memory_store.context", + "memory_store_batch.records.0.content", + "rule_update.patch.must_do", + "workspace_link.label", + } <= set(seeded), seeded diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 93ea35a..62d6ece 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -13,8 +13,11 @@ No target response may carry A's text, A's canary token, A's IDs or A's workspace ID; A's database may not change; a rejected call may not change the -target; and no string or key anywhere may carry a registered root or the home -directory. Every call's outcome is pinned. +target; and no string or key outside a ``UserText`` field may carry a +registered root or the home directory. Each target stores a note naming its +own root, which user text may do (UD-3), so the fields that echo it are found +from the same response models the server validates against, never by field +name. Every call's outcome is pinned. Pass 2 relaunches A and B over HTTP with a JWT principal granted only A. Every call that targets B, directly or as a consolidation or federation @@ -38,9 +41,18 @@ import pytest from fastmcp import Client from mcp.shared.exceptions import McpError +from pydantic import BaseModel, ValidationError +from daem0nmcp.api.v7.models import ApiResponse, RecordSummary, guarded_strings from daem0nmcp.api.v7.policy import V7_TOOL_LEVELS from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.tools import TOOL_DATA_MODELS, DiagnosticSummary from daem0nmcp.config import Settings from daem0nmcp.covenant import CovenantLevel from daem0nmcp.storage_activation import resolve_active_database @@ -94,6 +106,33 @@ Invoke = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] +RESOURCE_MODELS: dict[str, type[BaseModel]] = { + "warnings": WarningResourceDocument, + "failures": FailureResourceDocument, + "rules": RuleResourceDocument, + "active-context": ActiveContextResourceDocument, +} + + +def _response_model(tool: str) -> type[BaseModel]: + return ApiResponse[TOOL_DATA_MODELS[tool]] # type: ignore[name-defined] + + +def _host_path_leaks( + model: type[BaseModel], document: Any, needles: set[str] +) -> list[str]: + """Strings outside ``UserText`` fields that contain a host path needle. + + Validating against the response model also refuses any absolute path in + a server-generated field, whatever its spelling. + """ + validated = model.model_validate(document, strict=False) + return [ + value + for value in guarded_strings(validated) + if any(needle in value.lower() for needle in needles) + ] + def _placeholder(prefix: str) -> str: return f"{prefix}_{hashlib.sha256(prefix.encode()).hexdigest()}" @@ -641,6 +680,31 @@ def _is_protected(tool: str) -> bool: return V7_TOOL_LEVELS[tool] in {CovenantLevel.COUNSEL, CovenantLevel.DESTRUCTIVE} +def test_leak_check_exempts_user_text_by_annotation_only(tmp_path): + needles = {str(tmp_path).lower(), "alpha-private-root"} + summary = { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": f"Target note: see {tmp_path}", + "tags": ["/health"], + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + assert _host_path_leaks(RecordSummary, summary, needles) == [] + # The same root in a server-generated field is refused by the model... + with pytest.raises(ValidationError): + _host_path_leaks( + DiagnosticSummary, {"code": "X_Y", "message": f"at {tmp_path}"}, needles + ) + # ...and a spelling the path rule does not see is found by the scan. + leaked = {"code": "X_Y", "message": "at alpha-private-root"} + assert _host_path_leaks(DiagnosticSummary, leaked, needles) == [ + "at alpha-private-root" + ] + + def test_every_manifest_tool_is_swept_or_declared_cross_workspace(): swept = set(_sweep_arguments(Seed())) cross = set(CROSS_WORKSPACE_TOOLS) @@ -666,7 +730,9 @@ def __init__(self, roots: dict[str, Path], a_root: Path, seed: Seed, a: str): self.needles = needles | {Path(n).as_posix().lower() for n in needles} self.observed: dict[str, dict[str, list[str]]] = {} - def check(self, label: str, response: Any, request: Any) -> None: + def check( + self, label: str, response: Any, request: Any, model: type[BaseModel] + ) -> None: echoed = set(_strings(request)) for value in _strings(response): for secret in self.forbidden: @@ -674,19 +740,28 @@ def check(self, label: str, response: Any, request: Any) -> None: # A bare canary may appear only as an exact echo of the request # (memory_verify returns its claim). assert CANARY not in value or value in echoed, f"{label}: {value!r}" - lowered = value.lower() - for needle in self.needles: - assert needle not in lowered, f"{label}: host path in {value!r}" + leaks = _host_path_leaks(model, response, self.needles) + assert not leaks, f"{label}: host path in {leaks!r}" assert _ledger_counts(self.a_root) == self.a_ledger, f"{label} changed A" async def run(self, client, name: str, target: str, root: Path) -> None: invoke = _invoker(client) observed = self.observed.setdefault(name, {}) assert (await invoke("session_brief", {"workspace_id": target}))["ok"] + # The target's own note names its root. Reads echo it only in + # UserText fields; the leak check must pass without a name exemption. + note = { + "record_type": "warning", + "content": f"Target note: see {root / 'src' / 'neutral.py'}", + "idempotency_key": f"sweep-{name}-host-note-0001", + } + assert (await _protected(invoke, target, "memory_store", note))["ok"] own: dict[str, str] = {} for tool, arguments in OWN_VARIANTS: response = await _target_call(invoke, target, tool, arguments) - self.check(f"{name}:{tool} (own)", response, arguments) + self.check( + f"{name}:{tool} (own)", response, arguments, _response_model(tool) + ) own[tool] = _code(response) if tool == "code_search" and APPS_AVAILABLE: # The scan must cover real results, not an empty page. @@ -705,19 +780,16 @@ async def run(self, client, name: str, target: str, root: Path) -> None: for variant in variants: before = _ledger_counts(root) response = await _target_call(invoke, target, tool, variant) - self.check(f"{name}:{tool}", response, variant) + self.check(f"{name}:{tool}", response, variant, _response_model(tool)) if not response["ok"]: assert _ledger_counts(root) == before, f"{name}:{tool} wrote" observed.setdefault(tool, []).append(_code(response)) for tool, code in own.items(): observed[tool].append(code) - templates = [ - f"memory://workspaces/{target}/{kind}" - for kind in ("warnings", "failures", "rules", "active-context") - ] - for uri in templates: - contents = await client.read_resource(uri) - self.check(uri, [json.loads(item.text) for item in contents], uri) + for kind, model in RESOURCE_MODELS.items(): + uri = f"memory://workspaces/{target}/{kind}" + for item in await client.read_resource(uri): + self.check(uri, json.loads(item.text), uri, model) def assert_pinned(self, name: str) -> None: for tool, codes in sorted(self.observed[name].items()): @@ -802,7 +874,7 @@ async def _cross_workspace_checks(invoke, sweep, a, b, b_root, seed) -> list[str async def refused(tool, arguments, code): before = _ledger_counts(b_root) response = await _target_call(invoke, b, tool, arguments) - sweep.check(f"B:{tool}", response, arguments) + sweep.check(f"B:{tool}", response, arguments, _response_model(tool)) assert _code(response) == code, (tool, response) assert _ledger_counts(b_root) == before, tool From 9ef43d163c9ecd8f5be99a7f70bcbb4034a2ca9e Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:26:04 -0400 Subject: [PATCH 26/72] test: pin every read against its pre-path baseline Readers turn a row that fails its output model into CAPABILITY_DEGRADED, which the first version accepted as a gate. Compare each read with its outcome before any path was stored, count CAPABILITY_DEGRADED as a refusal for seeded calls, and retire the graph rebuild the inserted rows queue so entity IDs stay current. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_path_policy.py | 71 +++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py index f7df0f3..abf570d 100644 --- a/tests/api_v7/test_path_policy.py +++ b/tests/api_v7/test_path_policy.py @@ -63,7 +63,6 @@ from tests.api_v7.test_process_workspace_isolation_sweep import ( CANARY, CROSS_WORKSPACE_TOOLS, - GATE_CODES, PROFILE_ENVIRONMENT, _code, _is_protected, @@ -79,8 +78,15 @@ SEEDS = ("C:\\proj\\x.py", "/health", "\\d+") NEUTRAL = "plain neutral words" # A path in text is the user's business; these outcomes mean it was refused. +# Readers turn a row that fails its output model into CAPABILITY_DEGRADED. REFUSALS = frozenset( - {"INVALID_ARGUMENT", "INTERNAL_ERROR", "WORKSPACE_PATH_ESCAPE", "SCHEMA_REJECTED"} + { + "INVALID_ARGUMENT", + "INTERNAL_ERROR", + "WORKSPACE_PATH_ESCAPE", + "SCHEMA_REJECTED", + "CAPABILITY_DEGRADED", + } ) RESOURCE_KINDS = ("warnings", "failures", "rules", "active-context") TOKEN = "preflight-token-placeholder-0001" @@ -328,7 +334,9 @@ def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[ actor_type="migration", payload={ "trigger_id": "trg_" + _hex("trigger", text), - "trigger_type": "tag", + # A file glob: a tag pattern is a regex, which + # "C:\proj\x.py" is not. + "trigger_type": "file", "pattern": text, "recall_query": text, "categories": [], @@ -361,6 +369,15 @@ def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[ ), ), ) + # The graph rebuild queued by the events above just ran here; retire + # it so a later drain cannot swap the inserted entities for a new + # generation mid-test. + connection.execute( + "UPDATE background_jobs SET status='succeeded',updated_at_us=?," + "finished_at_us=? WHERE workspace_id=? AND status='queued' " + "AND idempotency_key='active-projection:graph'", + (_now_us(), _now_us(), workspace_id), + ) connection.commit() return record_ids @@ -556,12 +573,23 @@ async def _seed_free_text(invoke, workspace_id: str, tool: str, arguments: dict) return labels +async def _read_everything(invoke, workspace_id: str, seed) -> dict[str, str]: + """Every read tool with the workspace's own objects, then each resource.""" + codes: dict[str, str] = {} + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if _is_protected(tool) or tool in CROSS_WORKSPACE_TOOLS: + continue + plain = _fresh_keys(_own_read_arguments(arguments), f"{tool}-{len(codes)}") + codes[tool] = _code(await invoke(tool, {"workspace_id": workspace_id, **plain})) + return codes + + async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_path): """Seed every free-text input field and DB row with paths; read everything. - An input field typed as free text but not ``UserText`` refuses the seed - (``INVALID_ARGUMENT``), and an output field carrying it fails the read - (``INTERNAL_ERROR``); either fails this test. + An input field typed as free text but not ``UserText`` refuses the seed, + and an output field carrying it fails or degrades the read; either fails + this test. """ a_root, b_root = tmp_path / "alpha", tmp_path / "beta" alpha, beta = await initialize_workspaces((a_root, b_root)) @@ -576,6 +604,8 @@ async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_p invoke = _invoker(client) seed = await _seed_workspace_a(invoke, a, b) await _mint_tokens(invoke, a, b, seed) + # Outcomes before any path is stored. + baseline = await _read_everything(invoke, a, seed) writes = { **_sweep_arguments(seed), "workspace_link": {"linked_workspace_id": b}, @@ -593,23 +623,26 @@ async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_p entities = await invoke("entity_list", {"workspace_id": a, "limit": 100}) assert set(SEEDS) <= {item["name"] for item in entities["data"]["items"]} seed.ids["entity"] = entities["data"]["items"][0]["entity_id"] + # The inserted graph generation replaces the seed's community IDs. communities = await invoke("community_list", {"workspace_id": a}) - if communities["ok"] and communities["data"]["items"]: - seed.ids["community"] = communities["data"]["items"][0]["community_id"] - failures: dict[str, str] = {} + seed.ids["community"] = communities["data"]["items"][0]["community_id"] + after = await _read_everything(invoke, a, seed) for tool, arguments in sorted(_sweep_arguments(seed).items()): - if _is_protected(tool) or tool in CROSS_WORKSPACE_TOOLS: - continue - plain = _fresh_keys(_own_read_arguments(arguments), tool) - response = await invoke(tool, {"workspace_id": a, **plain}) - if _code(response) != "ok" and _code(response) not in GATE_CODES: - failures[tool] = _code(response) - # Query-like fields searching for the paths. - await _seed_free_text(invoke, a, tool, arguments) + if tool in after: + # Query-like fields searching for the paths. + await _seed_free_text(invoke, a, tool, arguments) for kind in RESOURCE_KINDS: await client.read_resource(f"memory://workspaces/{a}/{kind}") - # edit_preflight needs a native edit request, which only the edit hook makes. - assert failures == {"edit_preflight": "NOT_FOUND"}, failures + # Stored paths never turn a read into a failure. (Entity reads use a real + # entity only after the rows exist.) + changed = { + tool: (baseline[tool], code) + for tool, code in after.items() + if code not in {"ok", baseline[tool]} + } + assert not changed, changed + assert after["entity_evolution_trace"] == "ok", after + assert after["context_trigger_list"] == "ok", after # Nested and cross-workspace inputs are reached too. assert { "memory_store.context", From ce941433308eec4deff40616ec43df7d962a47b8 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 22:56:45 -0400 Subject: [PATCH 27/72] fix: keep contended reads and non-pinned tools retryable (review round 1) - resource_repository._run: a read that overran its deadline and then finished cleanly is contention, not a fault; report DATABASE_IN_USE instead of RESOURCE_REPOSITORY_UNAVAILABLE (-> PREFLIGHT_FAILED -> INTERNAL_ERROR), the chain behind the Windows CI failures (R-9, R-10). - application.py: non-pinned tools now send DATABASE_IN_USE as retryable with retry_after_ms=250, like pinned tools (R-1, R-15). - memory_store and token-bound non-pinned tools: the gate spends the preflight token before the write, so DATABASE_IN_USE carries a memory_preflight remedy and says to retry with a fresh token and the same idempotency_key (R-4). - is_database_busy uses sqlite_errorcode (BUSY/LOCKED) when available and SQLite's fixed lock messages on 3.10 (R-6); the retry hint is one constant. - governance_events INSERT: only IntegrityError is a stream conflict (R-2). - internal_error logs pydantic ValidationError links without input values (R-13); token-not-issued docstring states what is echoed (R-3/R-7/R-14). - Tests: service-level preflight/briefing contention (full pool, overrun read) through the production readers; repository overrun maps to DATABASE_IN_USE; same-token retry is TOKEN_REPLAYED with the preflight remedy; governance conflict rule; log redaction; server log tails in the two-process test's assertion messages; retry counts printed, not asserted (R-11, R-12). Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/application.py | 48 ++++++-- daem0nmcp/api/v7/errors.py | 22 +++- daem0nmcp/api/v7/pinned.py | 43 +++++-- daem0nmcp/api/v7/resource_repository.py | 7 +- daem0nmcp/api/v7/responses.py | 71 ++++++++++- daem0nmcp/event_store.py | 3 +- tests/api_v7/test_application.py | 53 +++++++++ tests/api_v7/test_error_honesty.py | 112 ++++++++++++++---- tests/api_v7/test_pinned_handlers.py | 18 ++- .../api_v7/test_process_concurrent_writers.py | 42 +++++-- tests/api_v7/test_resource_repository.py | 4 +- tests/test_event_store.py | 60 ++++++++++ 12 files changed, 408 insertions(+), 75 deletions(-) diff --git a/daem0nmcp/api/v7/application.py b/daem0nmcp/api/v7/application.py index 2104068..75a76f3 100644 --- a/daem0nmcp/api/v7/application.py +++ b/daem0nmcp/api/v7/application.py @@ -15,7 +15,7 @@ from ...covenant import CovenantGate, InvocationScope from ...workspace import Workspace -from .errors import ErrorCode +from .errors import DATABASE_IN_USE_RETRY_AFTER_MS, ErrorCode from .models import CapabilityState from .policy import V7_TOOL_LEVELS from .responses import ResponseFactory @@ -119,18 +119,26 @@ def _remedy( if code in {"COUNSEL_REQUIRED", "TOKEN_MISSING"} and isinstance( workspace_id, str ): - target_arguments = { - key: value - for key, value in effective.items() - if key not in {"workspace_id", "preflight_token"} - } - return "memory_preflight", { - "workspace_id": workspace_id, - "target_tool": tool_name, - "target_arguments": target_arguments, - } + return self._preflight_remedy(workspace_id, tool_name, effective) return None, {} + @staticmethod + def _preflight_remedy( + workspace_id: str, + tool_name: str, + effective: Mapping[str, Any], + ) -> tuple[str, dict[str, Any]]: + target_arguments = { + key: value + for key, value in effective.items() + if key not in {"workspace_id", "preflight_token"} + } + return "memory_preflight", { + "workspace_id": workspace_id, + "target_tool": tool_name, + "target_arguments": target_arguments, + } + def handler(self, tool_name: str) -> Callable[..., Any]: try: input_model = TOOL_INPUT_MODELS[tool_name] @@ -294,6 +302,24 @@ async def invoke(**arguments: Any) -> object: states = () if any(not isinstance(state, CapabilityState) for state in states): states = () + if error_code == ErrorCode.DATABASE_IN_USE.value: + # The gate already spent any preflight token, so a + # retry needs a fresh one. + workspace_id = effective.get("workspace_id") + remedy_tool, remedy_arguments = ( + self._preflight_remedy(workspace_id, tool_name, effective) + if isinstance(token, str) and isinstance(workspace_id, str) + else (None, {}) + ) + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool=remedy_tool, + remedy_arguments=remedy_arguments, + capability_states=tuple(states), + ) return response.failure( ErrorCode(error_code), "The operation could not be completed.", diff --git a/daem0nmcp/api/v7/errors.py b/daem0nmcp/api/v7/errors.py index b0e5820..f240673 100644 --- a/daem0nmcp/api/v7/errors.py +++ b/daem0nmcp/api/v7/errors.py @@ -66,23 +66,39 @@ def is_stable_error_code(value: object) -> bool: return isinstance(value, str) and value in STABLE_ERROR_CODE_SET +# Clients back off this long before retrying DATABASE_IN_USE. +DATABASE_IN_USE_RETRY_AFTER_MS = 250 + + def is_database_busy(error: BaseException | None) -> bool: """Return whether *error* is transient contention a caller may retry. A full worker pool and a SQLite lock or busy timeout both clear once the competing writer finishes, so they map to retryable ``DATABASE_IN_USE``. + The ``code`` check keeps an already-classified repository error (which is + not a ``RuntimeServiceError``) busy through the briefing/preflight wrappers. """ if isinstance(error, BoundedWorkerBusyError): return True if getattr(error, "code", None) == ErrorCode.DATABASE_IN_USE.value: return True - return isinstance(error, sqlite3.OperationalError) and any( - word in str(error).casefold() for word in ("locked", "busy") - ) + if not isinstance(error, sqlite3.OperationalError): + return False + code = getattr(error, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in {5, 6} # SQLITE_BUSY, SQLITE_LOCKED + # sqlite_errorcode is unavailable on Python 3.10; match SQLite's fixed + # lock diagnostics only, never arbitrary detail text. + return str(error).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } __all__ = [ + "DATABASE_IN_USE_RETRY_AFTER_MS", "ERROR_CODE_REGISTRY", "INTERNAL_ERROR_MESSAGE", "STABLE_ERROR_CODES", diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index b03bdb9..01ef10a 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -22,7 +22,7 @@ from ...event_store import AppendedEvent, EventStreamConflict from ...retrieval import RetrievalQuery from ...workspace import Workspace -from .errors import ErrorCode +from .errors import DATABASE_IN_USE_RETRY_AFTER_MS, ErrorCode from .models import ( ApiResponse, ApiWarning, @@ -129,8 +129,10 @@ def _path_safe_success( def _token_not_issued_warning(error: ArgumentNormalizationError) -> ApiWarning: """Say why a description-only preflight carries no token. - Only schema field names and pydantic error types are echoed; supplied - values and unknown keys never reach the wire. + Supplied values are never echoed: only pydantic error types and the + locations of the failing arguments. A location part is echoed only when + it is an int index or identifier-shaped (which may include a key the + caller sent); anything else, such as a path-like key, becomes ``?``. """ problems: list[str] = [] @@ -240,7 +242,11 @@ def _expected_service_failure( code, message, retryable=retryable, - retry_after_ms=250 if code is ErrorCode.DATABASE_IN_USE else None, + retry_after_ms=( + DATABASE_IN_USE_RETRY_AFTER_MS + if code is ErrorCode.DATABASE_IN_USE + else None + ), ) @@ -849,6 +855,14 @@ async def memory_store( if failure is not None: return failure assert scope is not None + preflight_remedy = { + "workspace_id": request.workspace_id, + "target_tool": "memory_store", + "target_arguments": request.model_dump( + mode="json", + exclude={"workspace_id", "preflight_token"}, + ), + } violation = self._dependencies.covenant_gate.authorize( "memory_store", request.model_dump(), @@ -875,14 +889,7 @@ async def memory_store( ErrorCode.TOKEN_LEGACY_UNSUPPORTED, }: remedy_tool = "memory_preflight" - remedy_arguments = { - "workspace_id": request.workspace_id, - "target_tool": "memory_store", - "target_arguments": request.model_dump( - mode="json", - exclude={"workspace_id", "preflight_token"}, - ), - } + remedy_arguments = preflight_remedy return response.failure( stable_code, "The preflight capability was rejected.", @@ -931,6 +938,18 @@ async def memory_store( retryable=True, ) except Exception as exc: + if getattr(exc, "code", None) == ErrorCode.DATABASE_IN_USE.value: + # authorize() already spent the token, so a retry needs a new + # one; the idempotency key keeps the retry replay-safe. + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use. Request a new " + "preflight token, then retry with the same idempotency_key.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool="memory_preflight", + remedy_arguments=preflight_remedy, + ) return _expected_service_failure(response, exc) or response.internal_error( exc ) diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index d6b839c..256a0c5 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -951,6 +951,7 @@ def _workspace_statistics_sync( async def _run(self, operation: Callable[[], T]) -> T: worker = asyncio.create_task(self._worker_pool.run(operation)) cancellation: asyncio.CancelledError | None = None + timed_out = False try: return cast( T, @@ -960,7 +961,7 @@ async def _run(self, operation: Callable[[], T]) -> T: ), ) except asyncio.TimeoutError: - pass + timed_out = True except asyncio.CancelledError as exc: cancellation = exc except Exception: @@ -985,7 +986,9 @@ async def _run(self, operation: Callable[[], T]) -> T: failure = worker.exception() if cancellation is not None: raise cancellation - if is_database_busy(failure): + # A read that overran its deadline but did not itself fail was slowed + # by contention (lock waits, a saturated host), so it is retryable. + if is_database_busy(failure) or (timed_out and failure is None): raise ResourceRepositoryError("DATABASE_IN_USE") from failure raise ResourceRepositoryError() from failure diff --git a/daem0nmcp/api/v7/responses.py b/daem0nmcp/api/v7/responses.py index 241b656..b913108 100644 --- a/daem0nmcp/api/v7/responses.py +++ b/daem0nmcp/api/v7/responses.py @@ -3,13 +3,17 @@ from __future__ import annotations import logging +import re import secrets import sys +import traceback from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, TypeVar +from pydantic import ValidationError + from .errors import INTERNAL_ERROR_MESSAGE, ErrorCode from .models import ( ApiError, @@ -25,6 +29,50 @@ T = TypeVar("T") _LOGGER = logging.getLogger(__name__) +_LOCATION_PART = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") + + +def _exception_chain(error: BaseException) -> list[BaseException]: + """Return *error* and its causes, root cause first, as tracebacks do.""" + + chain: list[BaseException] = [] + link: BaseException | None = error + while link is not None and all(link is not seen for seen in chain): + chain.append(link) + link = link.__cause__ or ( + None if link.__suppress_context__ else link.__context__ + ) + return chain[::-1] + + +def _without_input_values(chain: list[BaseException]) -> str: + """Render a traceback whose pydantic errors omit the offending values. + + ``str(ValidationError)`` quotes a slice of each invalid input, which for + service output can be stored memory content or a token. + """ + + rendered = [] + for link in chain: + if isinstance(link, ValidationError): + detail = "; ".join( + ".".join( + str(part) + if isinstance(part, int) or _LOCATION_PART.match(str(part)) + else "?" + for part in item["loc"] + ) + + f" ({item['type']})" + for item in link.errors(include_url=False) + ) + else: + detail = str(link) + rendered.append( + "Traceback (most recent call last):\n" + + "".join(traceback.format_tb(link.__traceback__)) + + f"{type(link).__module__}.{type(link).__qualname__}: {detail}" + ) + return "\n\nThe above exception led to:\n\n".join(rendered) def _utc_now() -> datetime: @@ -125,14 +173,25 @@ def internal_error(self, error: BaseException | None = None) -> ApiResponse[Any] """Return the one deliberately opaque internal failure envelope. The cause is logged (stderr by default, never the MCP wire) under the - correlation ID the caller receives, so the owner can find it. + correlation ID the caller receives, so the owner can find it. Logs + may still hold short exception messages such as local paths; redact + them before sharing. Pydantic input values are never logged. """ - _LOGGER.error( - "v7 internal error correlation_id=%s", - self.request_id, - exc_info=error if error is not None else sys.exc_info()[1], - ) + cause = error if error is not None else sys.exc_info()[1] + chain = [] if cause is None else _exception_chain(cause) + if any(isinstance(link, ValidationError) for link in chain): + _LOGGER.error( + "v7 internal error correlation_id=%s\n%s", + self.request_id, + _without_input_values(chain), + ) + else: + _LOGGER.error( + "v7 internal error correlation_id=%s", + self.request_id, + exc_info=cause, + ) return self.failure( ErrorCode.INTERNAL_ERROR, INTERNAL_ERROR_MESSAGE, diff --git a/daem0nmcp/event_store.py b/daem0nmcp/event_store.py index fd11307..e7328b6 100644 --- a/daem0nmcp/event_store.py +++ b/daem0nmcp/event_store.py @@ -1937,7 +1937,8 @@ def _append( "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # As for memory events: only a constraint collision is a conflict. raise EventStreamConflict( "governance event identity or stream version is occupied" ) from exc diff --git a/tests/api_v7/test_application.py b/tests/api_v7/test_application.py index 023a3d1..d20677d 100644 --- a/tests/api_v7/test_application.py +++ b/tests/api_v7/test_application.py @@ -129,6 +129,59 @@ async def operation(*, workspace: Workspace, request: object) -> object: self.assertEqual(second.error.code, "TOKEN_REPLAYED") self.assertEqual(len(calls), 1) + async def test_database_in_use_is_retryable_and_points_at_a_fresh_preflight( + self, + ) -> None: + from daem0nmcp.api.v7.models import MutationReceipt + + class BusyError(RuntimeError): + code = "DATABASE_IN_USE" + + attempts: list[object] = [] + + async def operation(*, workspace: Workspace, request: object) -> object: + del workspace + attempts.append(request) + if len(attempts) == 1: + raise BusyError("database is locked") + return MutationReceipt( + operation_id="op_archive_retry", + affected_ids=[request.record_id], + event_ids=[], + counts={"changed": 1}, + idempotent_replay=False, + ) + + self.gate.record_briefing(self.scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_id": RECORD_ID, + "archived": True, + } + handler = self._router({"memory_archive_set": operation}).handler( + "memory_archive_set" + ) + token = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + + busy = await handler(**target, preflight_token=token) + self.assertFalse(busy.ok) + self.assertEqual(busy.error.code, "DATABASE_IN_USE") + self.assertTrue(busy.error.retryable) + self.assertEqual(busy.error.retry_after_ms, 250) + self.assertEqual(busy.error.remedy.tool, "memory_preflight") + self.assertEqual( + busy.error.remedy.arguments["target_arguments"], + {"record_id": RECORD_ID, "archived": True}, + ) + # The token was spent before the operation ran: a literal retry is + # refused, and the remedy (a fresh preflight) succeeds. + replayed = await handler(**target, preflight_token=token) + self.assertEqual(replayed.error.code, "TOKEN_REPLAYED") + fresh = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + retried = await handler(**target, preflight_token=fresh) + self.assertTrue(retried.ok) + self.assertEqual(len(attempts), 2) + async def test_protected_durable_execution_requires_exact_worker_context( self, ) -> None: diff --git a/tests/api_v7/test_error_honesty.py b/tests/api_v7/test_error_honesty.py index 26b9743..0c1f407 100644 --- a/tests/api_v7/test_error_honesty.py +++ b/tests/api_v7/test_error_honesty.py @@ -15,12 +15,18 @@ from fastmcp import Client from daem0nmcp.api.v7 import runtime_services -from daem0nmcp.api.v7.production import create_v7_server -from daem0nmcp.api.v7.resource_repository import ( - ResourceRepositoryError, - SQLiteResourceRepository, +from daem0nmcp.api.v7.production import ( + _briefing_reader, + _guidance_reader, + create_v7_server, ) -from daem0nmcp.api.v7.resources import ResourceReadRequest +from daem0nmcp.api.v7.resource_repository import build_sqlite_resource_readers +from daem0nmcp.api.v7.runtime_services import ( + BasicBriefingService, + BasicPreflightService, + RuntimeServiceError, +) +from daem0nmcp.api.v7.tools import SessionBriefInput from daem0nmcp.bounded_workers import BoundedWorkerPool from daem0nmcp.config import Settings from daem0nmcp.storage_activation import resolve_active_database @@ -108,6 +114,8 @@ async def flow(index: int) -> list[dict[str, Any]]: for observed in results: assert observed[-1]["ok"], observed + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) with closing(sqlite3.connect(_database(tmp_path))) as connection: records = connection.execute("SELECT count(*) FROM memory_records").fetchone() events = connection.execute( @@ -166,27 +174,85 @@ def explode(*_arguments, **_keywords): assert "MEMORY_STORE_FAILED" in caplog.text -async def test_resource_reads_report_a_full_worker_pool_as_database_in_use(tmp_path): +@pytest.mark.parametrize("contention", ["full_pool", "overrun_read"]) +@pytest.mark.parametrize("service", ["memory_preflight", "session_brief"]) +async def test_contended_briefing_snapshot_is_retryable_database_in_use( + tmp_path, service, contention +): + """The CI failure path: guidance/briefing -> read_briefing_snapshot -> _run. + + MCP resource URIs still report an opaque access failure; the repository + code only matters on these two service paths. + """ [workspace] = await initialize_workspaces((tmp_path,)) - pool = BoundedWorkerPool(max_workers=1, thread_name_prefix="test-busy") + storage = tmp_path / ".daem0nmcp" / "storage" + pool = BoundedWorkerPool(max_workers=1, thread_name_prefix="test-contention") release = threading.Event() - occupant = asyncio.create_task(pool.run(lambda: release.wait(timeout=30))) - for _ in range(100): - if pool.in_flight: - break - await asyncio.sleep(0) - try: - repository = SQLiteResourceRepository( - lambda _workspace: pytest.fail("a full pool must not open storage"), - worker_pool=pool, + + def resolve(_workspace): + if contention == "overrun_read": + release.wait(timeout=30) # overrun the 0.05 s deadline, then succeed + return resolve_active_database(storage) + + readers = build_sqlite_resource_readers( + resolve, timeout_seconds=0.05, worker_pool=pool + ) + + async def pool_occupied() -> None: + deadline = asyncio.get_running_loop().time() + 10 + while not pool.in_flight: + assert asyncio.get_running_loop().time() < deadline + await asyncio.sleep(0.01) + + occupant = None + if contention == "full_pool": + occupant = asyncio.create_task(pool.run(lambda: release.wait(timeout=30))) + await pool_occupied() + if service == "memory_preflight": + call = BasicPreflightService(reader=_guidance_reader(readers)).guidance( + workspace, "memory_store", {}, "Plan a decision." + ) + else: + call = BasicBriefingService(reader=_briefing_reader(readers)).assemble( + workspace, SessionBriefInput(workspace_id=workspace.workspace_id) ) - with pytest.raises(ResourceRepositoryError) as caught: - await repository.read_warnings( - workspace, - ResourceReadRequest("warnings", 10, "updated_at_desc"), - ) - assert caught.value.code == "DATABASE_IN_USE" + task = asyncio.ensure_future(call) + try: + if contention == "overrun_read": + await pool_occupied() + await asyncio.sleep(0.2) + release.set() + with pytest.raises(RuntimeServiceError) as caught: + await task finally: release.set() - await occupant + if occupant is not None: + await occupant pool.shutdown() + assert caught.value.code == "DATABASE_IN_USE" + + +def test_internal_error_log_omits_pydantic_input_values(caplog): + from pydantic import BaseModel, ValidationError + + from daem0nmcp.api.v7.responses import ResponseFactory + + class Guidance(BaseModel): + priority: int + + # Built at runtime so the traceback's source lines cannot contain it. + stored_content = "-".join(("secret", "memory", "fragment", "canary")) + try: + try: + Guidance.model_validate({"priority": stored_content}) + except ValidationError as invalid: + raise RuntimeError("PREFLIGHT_FAILED") from invalid + except RuntimeError as error: + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = ResponseFactory().begin(None).internal_error(error) + + assert response.error.correlation_id in caplog.text + assert "priority (int_parsing)" in caplog.text + assert "PREFLIGHT_FAILED" in caplog.text + assert "test_internal_error_log_omits_pydantic_input_values" in caplog.text + assert "secret-memory-fragment-canary" not in caplog.text diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index 5622b20..d72cf31 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -1797,17 +1797,25 @@ def test_transient_writer_contention_is_retryable_database_in_use(self) -> None: ) ) - response = asyncio.run( - handlers["memory_store"]( - **MemoryStoreInput(**target, preflight_token=token).model_dump() - ) - ) + request = MemoryStoreInput(**target, preflight_token=token).model_dump() + response = asyncio.run(handlers["memory_store"](**request)) validated = MemoryStoreOutput.model_validate(response) self.assertFalse(validated.ok) self.assertEqual(ErrorCode.DATABASE_IN_USE, validated.error.code) self.assertTrue(validated.error.retryable) self.assertEqual(250, validated.error.retry_after_ms) + # authorize() spent the token before the write, so the documented + # retry is a fresh preflight with the same idempotency_key. + self.assertEqual("memory_preflight", validated.error.remedy.tool) + remedy_target = validated.error.remedy.arguments["target_arguments"] + self.assertEqual("decision-busy-1", remedy_target["idempotency_key"]) + self.assertNotIn("preflight_token", remedy_target) + replayed = MemoryStoreOutput.model_validate( + asyncio.run(handlers["memory_store"](**request)) + ) + self.assertEqual(ErrorCode.TOKEN_REPLAYED, replayed.error.code) + self.assertEqual("memory_preflight", replayed.error.remedy.tool) def test_record_outcome_maps_runtime_not_found(self) -> None: from daem0nmcp.api.v7 import pinned diff --git a/tests/api_v7/test_process_concurrent_writers.py b/tests/api_v7/test_process_concurrent_writers.py index 7010287..dc7b88b 100644 --- a/tests/api_v7/test_process_concurrent_writers.py +++ b/tests/api_v7/test_process_concurrent_writers.py @@ -4,7 +4,9 @@ import asyncio import sqlite3 +from collections.abc import Callable from contextlib import closing +from pathlib import Path from typing import Any from mcp import ClientSession @@ -14,6 +16,19 @@ SAME_KEY_STORES = 5 DISTINCT_KEY_STORES = 10 +LOG_NAMES = ("stdio-server-a.log", "stdio-server-b.log") + + +def _server_log_tails(root: Path, lines: int = 40) -> str: + """Server stderr (tracebacks by correlation ID) for assertion messages.""" + tails = [] + for name in LOG_NAMES: + try: + text = (root / name).read_text(encoding="utf-8", errors="replace") + except OSError as error: + text = f"" + tails.append(f"--- {name} ---\n" + "\n".join(text.splitlines()[-lines:])) + return "\n".join(tails) async def _store_until_ok( @@ -21,6 +36,7 @@ async def _store_until_ok( workspace_id: str, key: str, content: str, + diagnostics: Callable[[], str], ) -> list[dict[str, Any]]: """Preflight and store, retrying only what the server marks retryable.""" scope = {"workspace_id": workspace_id} @@ -43,9 +59,12 @@ async def _store_until_ok( observed.append(response) if response["ok"]: return observed - assert response["error"]["code"] == "DATABASE_IN_USE", observed - assert response["error"]["retryable"] is True, observed - assert asyncio.get_running_loop().time() < deadline, observed + retryable = ( + response["error"]["code"] == "DATABASE_IN_USE" + and response["error"]["retryable"] is True + ) + assert retryable, (observed, diagnostics()) + assert asyncio.get_running_loop().time() < deadline, (observed, diagnostics()) await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) @@ -53,8 +72,8 @@ async def test_two_stdio_servers_store_each_key_exactly_once(tmp_path): [workspace] = await initialize_workspaces((tmp_path,)) scope = {"workspace_id": workspace.workspace_id} async with ( - process_client(tmp_path, "stdio", log_name="stdio-server-a.log") as first, - process_client(tmp_path, "stdio", log_name="stdio-server-b.log") as second, + process_client(tmp_path, "stdio", log_name=LOG_NAMES[0]) as first, + process_client(tmp_path, "stdio", log_name=LOG_NAMES[1]) as second, ): sessions = (first, second) for session in sessions: @@ -68,17 +87,18 @@ async def test_two_stdio_servers_store_each_key_exactly_once(tmp_path): results = await asyncio.gather( *( _store_until_ok( - sessions[index % 2], workspace.workspace_id, key, content + sessions[index % 2], + workspace.workspace_id, + key, + content, + lambda: _server_log_tails(tmp_path), ) for index, (key, content) in enumerate(stores) ) ) - responses = [response for observed in results for response in observed] - assert all( - response["ok"] or response["error"]["code"] == "DATABASE_IN_USE" - for response in responses - ), responses + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) record_ids = {observed[-1]["data"]["record"]["record_id"] for observed in results} assert len(record_ids) == 1 + DISTINCT_KEY_STORES replays = [observed[-1]["data"]["idempotent_replay"] for observed in results] diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index 45a43a4..76d1e95 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -716,8 +716,10 @@ def resolve(_workspace: Workspace) -> ResolvedActiveDatabase: self.assertFalse(read.done()) self.assertEqual(pool.in_flight, 1) release.set() - with self.assertRaises(ResourceRepositoryError): + with self.assertRaises(ResourceRepositoryError) as caught: await read + # An overrun read that did not itself fail is retryable contention. + self.assertEqual(caught.exception.code, "DATABASE_IN_USE") self.assertEqual(pool.in_flight, 0) finally: release.set() diff --git a/tests/test_event_store.py b/tests/test_event_store.py index df75ba4..6e3066a 100644 --- a/tests/test_event_store.py +++ b/tests/test_event_store.py @@ -294,6 +294,66 @@ def __getattr__(self, name): store_type(failing).append_and_project(command) connection.rollback() + def test_only_integrity_errors_on_governance_insert_are_conflicts(self): + """The governance append follows the same conflict rule as memory.""" + from daem0nmcp.event_store import ( + EventStreamConflict, + GovernanceEventCommand, + GovernanceEventStore, + ) + from daem0nmcp.migrations.schema import MIGRATIONS + from daem0nmcp.schema_version import CURRENT_SCHEMA_VERSION + + connection = sqlite3.connect(":memory:") + self.addCleanup(connection.close) + connection.execute("PRAGMA foreign_keys=ON") + for version in range(16, CURRENT_SCHEMA_VERSION + 1): + for statement in next(m for m in MIGRATIONS if m[0] == version)[2]: + connection.execute(statement) + connection.commit() + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO governance_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = GovernanceEventCommand( + workspace_id=self.workspace_id, + stream_id="rule_" + "d" * 64, + stream_kind="rule", + event_type="rule.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + expected_stream_version=1, + payload={ + "rule_id": "rule_" + "d" * 64, + "trigger": "conflict mapping", + "must_do": [], + "must_not": [], + "ask_first": [], + "warnings": [], + "priority": 1, + "enabled": True, + "created_at_us": 1, + "updated_at_us": 1, + }, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + GovernanceEventStore(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + GovernanceEventStore(failing).append_and_project(command) + connection.rollback() + def test_projection_failure_rolls_back_event_with_savepoint(self): """No immutable event may survive a failed projection in caller transaction.""" command_type, store_type = self._api() From 134b72404f23bb7e2e662026266e54b8a9479cb8 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 23:04:06 -0400 Subject: [PATCH 28/72] fix: harden the remind-only hooks and the uninstall cleanup - pre_edit and stop find the nearest ancestor of cwd (then of CLAUDE_PROJECT_DIR) with .daem0nmcp/, never counting the home directory whose ~/.daem0nmcp/ is Daem0n's own state. - Model-supplied paths are checked lexically (normpath + commonpath; UNC and other-drive paths dropped) and never resolved, so a \host\share path cannot trigger SMB authentication. The path in the reminder is JSON-escaped. - stop JSON-parses only the transcript tail, keeps the full record count for its anti-loop state, prunes state files untouched for a week, and words its systemMessage for the user. - install/uninstall refuse a malformed ~/.claude/settings.json instead of replacing it; settings files with a UTF-8 BOM are read. - uninstall reads everything that can fail before writing, reports partial success accurately, only strips DAEM0NMCP_PROJECT_ROOT/STORAGE_PATH from a project pairing marked, checks each credential directory for links before removing it, and says when no project was given. The CLI defaults the project to the current directory and also accepts --project-path after the subcommand. Co-Authored-By: Claude Opus 5 --- daem0nmcp/claude_hooks/_client.py | 53 ++++++++ daem0nmcp/claude_hooks/install.py | 183 +++++++++++++++++---------- daem0nmcp/claude_hooks/pre_edit.py | 35 ++--- daem0nmcp/claude_hooks/stop.py | 107 +++++++++------- daem0nmcp/cli.py | 9 +- tests/api_v7/test_docs_hooks.py | 11 +- tests/test_claude_hooks_cli.py | 79 ++++++++++++ tests/test_claude_hooks_install.py | 110 ++++++++++++++++ tests/test_claude_hooks_post_edit.py | 30 ++++- tests/test_claude_hooks_pre_edit.py | 137 +++++++++++++------- tests/test_claude_hooks_stop.py | 20 ++- 11 files changed, 584 insertions(+), 190 deletions(-) diff --git a/daem0nmcp/claude_hooks/_client.py b/daem0nmcp/claude_hooks/_client.py index 53fd902..c123e22 100644 --- a/daem0nmcp/claude_hooks/_client.py +++ b/daem0nmcp/claude_hooks/_client.py @@ -33,6 +33,59 @@ def read_hook_event() -> dict[str, Any]: return value if isinstance(value, dict) else {} +def find_project_root(event: dict[str, Any]) -> Path | None: + """Return the nearest Daem0n project containing the event cwd. + + Tries ``cwd`` (which follows ``cd`` in Bash), then ``CLAUDE_PROJECT_DIR``, + walking up to the first ancestor with ``.daem0nmcp/``. The home directory + never counts: Daem0n keeps its own global state in ``~/.daem0nmcp/``. + """ + home = Path.home() + for raw in (event.get("cwd"), os.environ.get("CLAUDE_PROJECT_DIR")): + if not isinstance(raw, str) or not raw: + continue + start = Path(os.path.abspath(raw)) + for candidate in (start, *start.parents): + if candidate == home: + break + if (candidate / ".daem0nmcp").is_dir(): + return candidate + return None + + +def _is_unc(path: str) -> bool: + return path.startswith(("\\\\", "//")) + + +def relative_project_path( + project: Path, raw: object, base: object = None +) -> str | None: + """Return *raw* (relative to *base*, default *project*) relative to *project*. + + The result is in POSIX form, or None when *raw* is not inside *project*. + + Purely lexical: the path comes from the model, and resolving it on + Windows could open a ``\\\\host\\share`` path (SMB authentication) before + the user has approved anything. + """ + if not isinstance(raw, str) or not raw or "\0" in raw or _is_unc(raw): + return None + root = os.path.normpath(str(project)) + start = base if isinstance(base, str) and base else root + full = os.path.normpath(os.path.join(root, start, raw)) + if _is_unc(full): + return None + try: + inside = os.path.commonpath( + [os.path.normcase(root), os.path.normcase(full)] + ) == os.path.normcase(root) + except ValueError: # different drives, or mixed absolute/relative + return None + if not inside or os.path.normcase(full) == os.path.normcase(root): + return None + return Path(os.path.relpath(full, root)).as_posix() + + def get_project_path() -> str | None: """ Detect the project path from environment variables. diff --git a/daem0nmcp/claude_hooks/install.py b/daem0nmcp/claude_hooks/install.py index cd60be0..37fbd65 100644 --- a/daem0nmcp/claude_hooks/install.py +++ b/daem0nmcp/claude_hooks/install.py @@ -16,30 +16,48 @@ local_bridge_layout, provision_client_bridge_installation, ) +from ..protected_files import reject_linked_ancestry -# Every env key pairing can write into /.claude/settings.local.json. +# Env keys only pairing writes into /.claude/settings.local.json. +# Pairing always writes the credential key, so it marks a paired project. +_PAIRING_MARKER = "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" _BRIDGE_ENV_KEYS = ( - "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", + _PAIRING_MARKER, "DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR", "DAEM0NMCP_EDIT_BRIDGE_MODE", "DAEM0NMCP_EDIT_HOST_WORKSPACE_BINDING_FILE", - "DAEM0NMCP_PROJECT_ROOT", - "DAEM0NMCP_STORAGE_PATH", ) +# General Daem0n keys that pairing also sets; removed only from paired projects. +_SHARED_ENV_KEYS = ("DAEM0NMCP_PROJECT_ROOT", "DAEM0NMCP_STORAGE_PATH") def _settings_path() -> Path: return Path.home() / ".claude" / "settings.json" +def _read_json_object(path: Path) -> dict[str, Any]: + """Read a JSON object (a UTF-8 BOM is fine); raise ValueError otherwise.""" + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{path} is not readable JSON ({exc})") from exc + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + def _read_settings() -> dict[str, Any]: + """Read ``~/.claude/settings.json``; raise ValueError rather than lose it.""" path = _settings_path() - if path.exists(): - try: - return json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return {} - return {} + if not path.exists(): + return {} + settings = _read_json_object(path) + hooks = settings.get("hooks", {}) + if not isinstance(hooks, dict) or not all( + isinstance(entries, list) for entries in hooks.values() + ): + raise ValueError(f"{path}: 'hooks' must map each event to a list") + return settings def _write_settings(data: dict[str, Any]) -> None: @@ -50,8 +68,11 @@ def _write_settings(data: dict[str, Any]) -> None: def _is_daem0n_entry(entry: dict) -> bool: """Check if a hook entry belongs to Daem0n (current or legacy).""" - for hook in entry.get("hooks", []): - cmd = hook.get("command", "") + hooks = entry.get("hooks") if isinstance(entry, dict) else None + for hook in hooks if isinstance(hooks, list) else []: + cmd = hook.get("command") if isinstance(hook, dict) else None + if not isinstance(cmd, str): + continue if "daem0nmcp.claude_hooks" in cmd: return True # Legacy hook scripts @@ -155,12 +176,7 @@ def _configure_project_bridge( environment: dict[str, str], ) -> Path: path = project_root / ".claude" / "settings.local.json" - if path.exists(): - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError("project-local Claude settings must be an object") - else: - value = {} + value = _read_json_object(path) if path.exists() else {} configured = value.setdefault("env", {}) if not isinstance(configured, dict): raise ValueError("project-local Claude env settings must be an object") @@ -189,7 +205,10 @@ def install_claude_hooks( Returns (success, message). """ - settings = _read_settings() + try: + settings = _read_settings() + except ValueError as exc: + return False, f"{exc}; fix or move it and retry. Nothing was changed." new_defs = _build_hook_definitions() project_root = ( None if project_path is None else Path(project_path).resolve(strict=True) @@ -250,25 +269,19 @@ def install_claude_hooks( return True, message -def _unconfigure_project_bridge(project_root: Path, dry_run: bool) -> Path | None: - """Strip the env keys pairing added; return the settings path if changed.""" - path = project_root / ".claude" / "settings.local.json" +def _strip_pairing_env(path: Path) -> dict[str, Any] | None: + """Return *path*'s settings without the pairing env keys, or None if unpaired.""" if not path.is_file(): return None - value = json.loads(path.read_text(encoding="utf-8")) - configured = value.get("env") if isinstance(value, dict) else None - if not isinstance(configured, dict): - return None - removed = [key for key in _BRIDGE_ENV_KEYS if key in configured] - if not removed: + value = _read_json_object(path) + configured = value.get("env") + if not isinstance(configured, dict) or _PAIRING_MARKER not in configured: return None - for key in removed: - del configured[key] + for key in _BRIDGE_ENV_KEYS + _SHARED_ENV_KEYS: + configured.pop(key, None) if not configured: del value["env"] - if not dry_run: - path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") - return path + return value def uninstall_claude_hooks( @@ -286,12 +299,45 @@ def uninstall_claude_hooks( ``.claude/settings.local.json``; with *remove_credentials*, also deletes the project's local edit-bridge credential and socket directories. - Returns (success, message). + Returns (success, message); the message lists everything that was done, + including after a partial failure. """ - settings = _read_settings() + try: + settings = _read_settings() + except ValueError as exc: + return False, f"{exc}; fix or move it and retry. Nothing was changed." + + # Read everything that can fail before writing anything. + notes: list[str] = [] + ok = True + local_path: Path | None = None + local_value: dict[str, Any] | None = None + directories: list[Path] = [] + if project_path is not None: + project_root = Path(project_path).resolve(strict=False) + local_path = project_root / ".claude" / "settings.local.json" + try: + local_value = _strip_pairing_env(local_path) + except ValueError as exc: + notes.append(f"Left {local_path} unchanged: {exc}") + ok = False + if remove_credentials: + try: + _, _, *layout = local_bridge_layout(project_root, bridge_config_root) + except (OSError, ValueError) as exc: + notes.append( + f"Could not locate edit bridge credentials for {project_root} " + f"({exc}); delete its directory under " + "~/.daem0nmcp/edit-bridges/ by hand." + ) + ok = False + else: + directories = [path for path in layout if path.is_dir()] + elif remove_credentials: + notes.append("No project given; edit bridge credentials kept.") + hooks_section = settings.get("hooks", {}) removed_events: list[str] = [] - for event in list(hooks_section.keys()): entries = hooks_section[event] filtered = [e for e in entries if not _is_daem0n_entry(e)] @@ -303,42 +349,43 @@ def uninstall_claude_hooks( del hooks_section[event] lines: list[str] = [] - if removed_events: + if not removed_events: + lines.append("No Daem0n hooks found to remove.") + elif dry_run: settings["hooks"] = hooks_section - if dry_run: - formatted = json.dumps(settings, indent=2) - lines.append(f"[dry-run] Would write to {_settings_path()}:\n{formatted}") - else: - try: - _write_settings(settings) - except OSError as exc: - return False, f"Failed to write settings: {exc}" - lines.append( - f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}" - ) + formatted = json.dumps(settings, indent=2) + lines.append(f"[dry-run] Would write to {_settings_path()}:\n{formatted}") else: - lines.append("No Daem0n hooks found to remove.") - - if project_path is not None: - prefix = "[dry-run] Would remove" if dry_run else "Removed" + settings["hooks"] = hooks_section + try: + _write_settings(settings) + except OSError as exc: + return False, f"Failed to write settings: {exc}" + lines.append(f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}") + lines.extend(notes) + + prefix = "[dry-run] Would remove" if dry_run else "Removed" + if local_path is not None and local_value is not None: try: - project_root = Path(project_path).resolve(strict=True) - changed = _unconfigure_project_bridge(project_root, dry_run) - if changed is not None: - lines.append(f"{prefix} edit bridge settings from {changed}") - if remove_credentials: - _, _, *directories = local_bridge_layout( - project_root, bridge_config_root + if not dry_run: + local_path.write_text( + json.dumps(local_value, indent=2) + "\n", encoding="utf-8" ) - for directory in directories: - if directory.is_dir(): - if not dry_run: - shutil.rmtree(directory) - lines.append(f"{prefix} {directory}") - except (OSError, ValueError, json.JSONDecodeError) as exc: - return False, f"Edit bridge cleanup failed: {exc}" - - return True, "\n".join(lines) + lines.append(f"{prefix} edit bridge settings from {local_path}") + except OSError as exc: + lines.append(f"Could not update {local_path}: {exc}") + ok = False + for directory in directories: + try: + reject_linked_ancestry(directory) + if not dry_run: + shutil.rmtree(directory) + lines.append(f"{prefix} {directory}") + except OSError as exc: + lines.append(f"Could not remove {directory}: {exc}") + ok = False + + return ok, "\n".join(lines) if __name__ == "__main__": diff --git a/daem0nmcp/claude_hooks/pre_edit.py b/daem0nmcp/claude_hooks/pre_edit.py index fc34874..b182b89 100644 --- a/daem0nmcp/claude_hooks/pre_edit.py +++ b/daem0nmcp/claude_hooks/pre_edit.py @@ -1,6 +1,6 @@ """Claude Code PreToolUse hook for edits: remind, never block. -Stdlib only (plus ``read_hook_event``) so it stays fast on every edit. Outside +Stdlib only (plus the ``_client`` helpers) so it stays fast on every edit. Outside a Daem0n project it is silent; inside one it adds a one-line reminder to the model's context and always exits 0. """ @@ -8,36 +8,25 @@ from __future__ import annotations import json -import os import sys -from pathlib import Path -from ._client import read_hook_event - - -def _relative_path(project: Path, tool_input: object) -> str | None: - if not isinstance(tool_input, dict): - return None - raw = tool_input.get("file_path") or tool_input.get("notebook_path") - if not isinstance(raw, str) or not raw: - return None - try: - return (project / raw).resolve().relative_to(project.resolve()).as_posix() - except (OSError, ValueError): - return None +from ._client import find_project_root, read_hook_event, relative_project_path def reminder(event: dict) -> str | None: """Return the reminder JSON for this event, or None to stay silent.""" - root = event.get("cwd") or os.environ.get("CLAUDE_PROJECT_DIR") - if not isinstance(root, str) or not root: + project = find_project_root(event) + if project is None: return None - project = Path(root) - if not (project / ".daem0nmcp").is_dir(): - return None - path = _relative_path(project, event.get("tool_input")) + tool_input = event.get("tool_input") + raw = ( + tool_input.get("file_path") or tool_input.get("notebook_path") + if isinstance(tool_input, dict) + else None + ) + path = relative_project_path(project, raw, event.get("cwd")) recall = ( - f'memory_recall_file(relative_file_path="{path}")' + f"memory_recall_file(relative_file_path={json.dumps(path)})" if path else "memory_recall_file" ) diff --git a/daem0nmcp/claude_hooks/stop.py b/daem0nmcp/claude_hooks/stop.py index c40d93e..609836f 100644 --- a/daem0nmcp/claude_hooks/stop.py +++ b/daem0nmcp/claude_hooks/stop.py @@ -9,13 +9,19 @@ import contextlib import hashlib import json -import os import re import sys +import time +from collections import deque from pathlib import Path from ..workspace import WorkspaceRegistry -from ._client import read_hook_event, run_async +from ._client import ( + find_project_root, + read_hook_event, + relative_project_path, + run_async, +) # ─── transcript analysis ─────────────────────────────────────────── @@ -102,37 +108,56 @@ def _load_state(session_id: str) -> dict: return {"reminder_count": 0, "last_reminder_turn": -1} +_STATE_MAX_AGE_SECONDS = 7 * 24 * 3600 + + def _save_state(session_id: str, state: dict) -> None: with contextlib.suppress(OSError): - _state_file(session_id).write_text(json.dumps(state), encoding="utf-8") + target = _state_file(session_id) + target.write_text(json.dumps(state), encoding="utf-8") + # One file per session: drop the ones no session has touched in a week. + cutoff = time.time() - _STATE_MAX_AGE_SECONDS + for old in target.parent.glob("stop_*.json"): + with contextlib.suppress(OSError): + if old.stat().st_mtime < cutoff: + old.unlink() # ─── transcript reading ─────────────────────────────────────────── -def _read_transcript(path: object) -> list[dict]: - """Read Claude Code JSONL records, unwrapping the nested ``message``.""" +_TRANSCRIPT_TAIL_RECORDS = 50 + + +def _read_transcript(path: object) -> tuple[list[dict], int]: + """Return the last Claude Code JSONL records and the total record count. + + Only the tail is JSON-parsed (the analysis looks at the last 10 records); + nested ``message`` objects are unwrapped. + """ if not isinstance(path, str) or not path or not Path(path).is_file(): - return [] - messages = [] + return [], 0 + tail: deque[str] = deque(maxlen=_TRANSCRIPT_TAIL_RECORDS) + total = 0 try: - with open(path, encoding="utf-8") as f: + with open(path, encoding="utf-8", errors="replace") as f: for line in f: - line = line.strip() - if line: - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(record, dict) and isinstance( - record.get("message"), dict - ): - record = record["message"] - if isinstance(record, dict): - messages.append(record) + if line.strip(): + total += 1 + tail.append(line) except OSError: - pass - return messages + return [], 0 + messages = [] + for line in tail: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("message"), dict): + record = record["message"] + if isinstance(record, dict): + messages.append(record) + return messages, total def _get_recent_assistant_content(messages: list[dict], lookback: int = 5) -> str: @@ -222,14 +247,7 @@ def _workspace_id(project_path: str) -> str: def _relative_record_path(project_path: str, mentioned_path: str | None) -> str | None: if not mentioned_path: return None - try: - root = Path(project_path).resolve(strict=True) - candidate = Path(mentioned_path) - if not candidate.is_absolute(): - candidate = root / candidate - return candidate.resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None + return relative_project_path(Path(project_path), mentioned_path) def _memory_store_suggestion( @@ -293,6 +311,7 @@ async def analyse_and_remember( project_path: str, messages: list[dict], state: dict, + turn: int | None = None, ) -> StopResult: """ Core logic extracted for in-process testing. @@ -302,7 +321,7 @@ async def analyse_and_remember( * Returns result message (empty = nothing to say) * Updates *state* in-place for anti-loop tracking """ - current_turn = len(messages) + current_turn = len(messages) if turn is None else turn # Anti-loop check reminded_recently = state.get("last_reminder_turn", -1) >= current_turn - 2 @@ -337,10 +356,11 @@ async def analyse_and_remember( ) return StopResult( message=( - "[Daem0n suggests] Completion detected. The hook did not write " - "memory. Review each extracted decision, then execute:\n" + "Daem0n: this task looks finished and no outcome was recorded. " + "The hook wrote nothing. To keep these decisions, ask Claude " + "to run:\n" f"{suggestions}\n" - "When results are known, call " + "Once the result is known, ask Claude to call " "mcp__daem0nmcp__memory_record_outcome(" f'workspace_id="{workspace_id}", record_id="", ' 'outcome_text="", worked=true, ' @@ -350,10 +370,10 @@ async def analyse_and_remember( return StopResult( message=( - "[Daem0n whispers] Task completion detected. " - "If you made a durable decision, use memory_preflight for the exact " - "memory_store arguments before writing it. When a stored result is " - "known, call mcp__daem0nmcp__memory_record_outcome(" + "Daem0n: this task looks finished and no outcome was recorded. " + "The hook wrote nothing. To keep a durable decision, ask Claude to " + "run memory_preflight and then memory_store for it. Once the result " + "is known, ask Claude to call mcp__daem0nmcp__memory_record_outcome(" f'workspace_id="{workspace_id}", record_id="", ' 'outcome_text="", worked=true, ' f'idempotency_key="{outcome_key}").' @@ -366,13 +386,12 @@ async def analyse_and_remember( def main() -> None: event = read_hook_event() - project_path = event.get("cwd") or os.environ.get("CLAUDE_PROJECT_DIR") - if not isinstance(project_path, str) or not project_path: - sys.exit(0) - if not (Path(project_path) / ".daem0nmcp").is_dir(): + project = find_project_root(event) + if project is None: sys.exit(0) + project_path = str(project) - messages = _read_transcript(event.get("transcript_path")) + messages, turn = _read_transcript(event.get("transcript_path")) if not messages: sys.exit(0) @@ -380,7 +399,7 @@ def main() -> None: if not isinstance(session_id, str) or not session_id: session_id = "default" state = _load_state(session_id) - result = run_async(analyse_and_remember(project_path, messages, state)) + result = run_async(analyse_and_remember(project_path, messages, state, turn)) _save_state(session_id, state) if result.message: diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index 05b3c6a..08aae92 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -473,8 +473,13 @@ def main(): uninstall_claude_parser.add_argument( "--remove-credentials", action="store_true", - help="Also delete the project's local edit-bridge credential (with --project-path)", + help="Also delete the project's local edit-bridge credential", ) + for claude_parser in (install_claude_parser, uninstall_claude_parser): + # Also accept --project-path after the subcommand (default: cwd). + claude_parser.add_argument( + "--project-path", default=argparse.SUPPRESS, help="Project root path" + ) # install-opencode command install_oc_parser = subparsers.add_parser( @@ -935,7 +940,7 @@ def main(): success, message = uninstall_claude_hooks( dry_run=getattr(args, "dry_run", False), - project_path=args.project_path, + project_path=args.project_path or os.getcwd(), remove_credentials=args.remove_credentials, ) if args.json: diff --git a/tests/api_v7/test_docs_hooks.py b/tests/api_v7/test_docs_hooks.py index dff4413..3fef514 100644 --- a/tests/api_v7/test_docs_hooks.py +++ b/tests/api_v7/test_docs_hooks.py @@ -10,6 +10,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -295,8 +296,14 @@ async def test_stop_suggests_replay_safe_v7_calls_without_writing(self) -> None: def test_pre_edit_reminds_without_blocking(self) -> None: from daem0nmcp.claude_hooks.pre_edit import reminder - with tempfile.TemporaryDirectory() as tmp_dir: - project = Path(tmp_dir) + with ( + tempfile.TemporaryDirectory() as tmp_dir, + # Bound the upward project search to the temp dir. + mock.patch.object(Path, "home", return_value=Path(tmp_dir)), + mock.patch.dict(os.environ, {"CLAUDE_PROJECT_DIR": ""}), + ): + project = Path(tmp_dir) / "project" + project.mkdir() event = { "cwd": str(project), "tool_name": "Edit", diff --git a/tests/test_claude_hooks_cli.py b/tests/test_claude_hooks_cli.py index 18ba48a..2eaf291 100644 --- a/tests/test_claude_hooks_cli.py +++ b/tests/test_claude_hooks_cli.py @@ -42,3 +42,82 @@ def test_cli_uninstall_claude_hooks_dry_run(fake_settings): ok, msg = uninstall_claude_hooks(dry_run=True) assert ok assert "No Daem0n hooks found" in msg + + +def _cli(monkeypatch, capsys, *argv: str) -> tuple[int, str]: + from daem0nmcp import cli + + monkeypatch.setattr("sys.argv", ["daem0nmcp.cli", *argv]) + with pytest.raises(SystemExit) as exc_info: + cli.main() + return exc_info.value.code, capsys.readouterr().out + + +def test_cli_uninstall_accepts_project_path_after_the_subcommand( + fake_settings, tmp_path, monkeypatch, capsys +): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("DAEM0NMCP_PROJECT_ROOT", "unset-by-test") + project = tmp_path / "project" + project.mkdir() + + code, _out = _cli( + monkeypatch, capsys, "--project-path", str(project), "install-claude-hooks" + ) + assert code == 0 + bridges = home / ".daem0nmcp" / "edit-bridges" + credential_dirs = [p for p in bridges.iterdir() if p.name != "run"] + assert len(credential_dirs) == 1 + + code, out = _cli( + monkeypatch, + capsys, + "uninstall-claude-hooks", + "--project-path", + str(project), + "--dry-run", + "--remove-credentials", + ) + assert code == 0 + assert f"[dry-run] Would remove {credential_dirs[0]}" in out + assert credential_dirs[0].exists() + + code, out = _cli( + monkeypatch, + capsys, + "uninstall-claude-hooks", + "--project-path", + str(project), + "--remove-credentials", + ) + assert code == 0, out + assert not credential_dirs[0].exists() + local = json.loads( + (project / ".claude" / "settings.local.json").read_text(encoding="utf-8") + ) + assert local == {} + + +def test_cli_remove_credentials_defaults_to_the_current_directory( + fake_settings, tmp_path, monkeypatch, capsys +): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + + assert _cli(monkeypatch, capsys, "install-claude-hooks")[0] == 0 + code, out = _cli( + monkeypatch, capsys, "uninstall-claude-hooks", "--remove-credentials" + ) + + assert code == 0, out + assert "edit-bridges" in out + bridges = home / ".daem0nmcp" / "edit-bridges" + assert [p for p in bridges.iterdir() if p.name != "run"] == [] diff --git a/tests/test_claude_hooks_install.py b/tests/test_claude_hooks_install.py index 17a1676..a44444e 100644 --- a/tests/test_claude_hooks_install.py +++ b/tests/test_claude_hooks_install.py @@ -1,6 +1,7 @@ """Tests for the Claude Code hook installer.""" import json +import shutil import ssl from pathlib import Path @@ -391,3 +392,112 @@ def test_project_cleanup_without_credentials_keeps_them( assert ok, message assert json.loads(local_settings.read_text(encoding="utf-8")) == {} assert credential.is_file() + + +def _paired_project(tmp_path): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + ok, message = install_claude_hooks( + project_path=project, bridge_config_root=host_config + ) + assert ok, message + local_settings = project / ".claude" / "settings.local.json" + env = json.loads(local_settings.read_text(encoding="utf-8"))["env"] + credential_dir = Path(env["DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE"]).parent + runtime = Path(env["DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR"]) + runtime.mkdir(parents=True) + return project, host_config, local_settings, credential_dir, runtime + + +class TestUninstallRobustness: + def test_unpaired_project_keeps_general_daem0n_keys(self, fake_settings, tmp_path): + project = tmp_path / "project" + (project / ".claude").mkdir(parents=True) + local_settings = project / ".claude" / "settings.local.json" + original = json.dumps( + {"env": {"DAEM0NMCP_PROJECT_ROOT": "x", "DAEM0NMCP_STORAGE_PATH": "y"}} + ) + local_settings.write_text(original, encoding="utf-8") + + ok, _message = uninstall_claude_hooks(project_path=project) + + assert ok + assert local_settings.read_text(encoding="utf-8") == original + + def test_bom_prefixed_local_settings_are_cleaned(self, fake_settings, tmp_path): + project, _host, local_settings, _cred, _runtime = _paired_project(tmp_path) + text = local_settings.read_text(encoding="utf-8") + local_settings.write_bytes(b"\xef\xbb\xbf" + text.encode("utf-8")) + + ok, message = uninstall_claude_hooks(project_path=project) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == {} + + def test_malformed_local_settings_report_partial_success( + self, fake_settings, tmp_path + ): + project, host, local_settings, credential_dir, runtime = _paired_project( + tmp_path + ) + local_settings.write_bytes(b'{"env": {bad json') + + ok, message = uninstall_claude_hooks( + project_path=project, remove_credentials=True, bridge_config_root=host + ) + + assert not ok + assert "Removed Daem0n hooks from" in message + assert f"Left {local_settings.resolve()} unchanged" in message + assert local_settings.read_bytes() == b'{"env": {bad json' + assert json.loads(fake_settings.read_text())["hooks"] == {} + assert not credential_dir.exists() and not runtime.exists() + + def test_deleted_project_reports_what_was_and_was_not_done( + self, fake_settings, tmp_path + ): + project, host, _local, credential_dir, _runtime = _paired_project(tmp_path) + shutil.rmtree(project) + + ok, message = uninstall_claude_hooks( + project_path=project, remove_credentials=True, bridge_config_root=host + ) + + assert not ok + assert "Removed Daem0n hooks from" in message + assert "Could not locate edit bridge credentials" in message + assert "by hand" in message + assert credential_dir.exists() + + def test_remove_credentials_without_project_says_so(self, fake_settings): + ok, message = uninstall_claude_hooks(remove_credentials=True) + assert ok + assert "No project given; edit bridge credentials kept." in message + + @pytest.mark.parametrize( + "content", + ['{bad json, "theme": "dark"', '{"hooks": []}', '{"hooks": {"Stop": {}}}'], + ) + def test_malformed_user_settings_are_never_overwritten( + self, fake_settings, content + ): + fake_settings.write_text(content, encoding="utf-8") + + installed, install_message = install_claude_hooks() + removed, uninstall_message = uninstall_claude_hooks() + + assert not installed and not removed + assert "Nothing was changed" in install_message + assert "Nothing was changed" in uninstall_message + assert fake_settings.read_text(encoding="utf-8") == content + + def test_non_object_hook_entries_are_kept(self, fake_settings): + fake_settings.write_text( + json.dumps({"hooks": {"Stop": ["odd", {"hooks": "odd"}]}}), + encoding="utf-8", + ) + ok, message = install_claude_hooks() + assert ok, message + stop = json.loads(fake_settings.read_text())["hooks"]["Stop"] + assert stop[:2] == ["odd", {"hooks": "odd"}] diff --git a/tests/test_claude_hooks_post_edit.py b/tests/test_claude_hooks_post_edit.py index 693bab8..aab0fd6 100644 --- a/tests/test_claude_hooks_post_edit.py +++ b/tests/test_claude_hooks_post_edit.py @@ -52,7 +52,7 @@ def test_unpaired_project_exits_zero_without_importing_the_bridge(tmp_path): assert result.stdout.split() == ["0", "False"] -def test_paired_project_without_consumed_edit_exits_zero_silently(tmp_path): +def test_misconfigured_credential_still_exits_zero(tmp_path): (tmp_path / ".daem0nmcp").mkdir() result = _run( _event(tmp_path), @@ -60,3 +60,31 @@ def test_paired_project_without_consumed_edit_exits_zero_silently(tmp_path): ) assert result.returncode == 0, result.stderr assert result.stdout.split() == ["0", "True"] + + +def test_paired_project_without_consumed_edit_captures_nothing(tmp_path, monkeypatch): + from daem0nmcp.claude_hooks.post_edit import handle_post_edit + from daem0nmcp.edit_host import ( + EditHostStateStore, + provision_local_bridge_installation, + ) + + project = tmp_path / "project" + (project / ".daem0nmcp").mkdir(parents=True) + installation = provision_local_bridge_installation( + project, config_root=tmp_path / "host" + ) + for name, value in installation.environment().items(): + monkeypatch.setenv(name, value) + lookups = [] + original = EditHostStateStore.get_consumed + + def spy(self, **kwargs): + result = original(self, **kwargs) + lookups.append(result) + return result + + monkeypatch.setattr(EditHostStateStore, "get_consumed", spy) + + assert handle_post_edit(_event(project), str(project)) is False + assert lookups == [None] diff --git a/tests/test_claude_hooks_pre_edit.py b/tests/test_claude_hooks_pre_edit.py index 558bc49..c6fc1cf 100644 --- a/tests/test_claude_hooks_pre_edit.py +++ b/tests/test_claude_hooks_pre_edit.py @@ -6,16 +6,29 @@ import sys from pathlib import Path +import pytest + +from daem0nmcp.claude_hooks._client import relative_project_path + ROOT = Path(__file__).resolve().parents[1] -def _run(args: list[str], event: dict, env: dict | None = None): - environment = {**os.environ, "PYTHONPATH": str(ROOT)} +def _run( + args: list[str], event: dict | str, home: Path, env: dict | None = None +) -> subprocess.CompletedProcess[str]: + # HOME bounds the project search, so an enclosing checkout's .daem0nmcp/ + # (or the developer's CLAUDE_PROJECT_DIR) can never leak into a test. + environment = { + **os.environ, + "PYTHONPATH": str(ROOT), + "HOME": str(home), + "USERPROFILE": str(home), + } environment.pop("CLAUDE_PROJECT_DIR", None) environment.update(env or {}) return subprocess.run( [sys.executable, *args], - input=json.dumps(event), + input=event if isinstance(event, str) else json.dumps(event), capture_output=True, text=True, env=environment, @@ -24,6 +37,10 @@ def _run(args: list[str], event: dict, env: dict | None = None): ) +def _pre_edit(event: dict | str, home: Path, env: dict | None = None): + return _run(["-m", "daem0nmcp.claude_hooks.pre_edit"], event, home, env) + + def _write_event(cwd: Path, file_path: str) -> dict: return { "session_id": "s-1", @@ -36,24 +53,29 @@ def _write_event(cwd: Path, file_path: str) -> dict: } +def _context(result: subprocess.CompletedProcess[str]) -> str: + return json.loads(result.stdout)["hookSpecificOutput"]["additionalContext"] + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + root = tmp_path / "project" + (root / ".daem0nmcp").mkdir(parents=True) + return root + + def test_non_daem0n_directory_exits_zero_silently(tmp_path): - result = _run( - ["-m", "daem0nmcp.claude_hooks.pre_edit"], - _write_event(tmp_path, str(tmp_path / "app.py")), - ) + plain = tmp_path / "plain" + plain.mkdir() + result = _pre_edit(_write_event(plain, str(plain / "app.py")), tmp_path) assert result.returncode == 0 assert result.stdout == "" -def test_daem0n_project_exits_zero_with_reminder(tmp_path): - (tmp_path / ".daem0nmcp").mkdir() - result = _run( - ["-m", "daem0nmcp.claude_hooks.pre_edit"], - _write_event(tmp_path, str(tmp_path / "src" / "app.py")), - ) +def test_daem0n_project_exits_zero_with_reminder(tmp_path, project): + result = _pre_edit(_write_event(project, str(project / "src" / "app.py")), tmp_path) assert result.returncode == 0 - payload = json.loads(result.stdout) - output = payload["hookSpecificOutput"] + output = json.loads(result.stdout)["hookSpecificOutput"] assert output["hookEventName"] == "PreToolUse" assert "permissionDecision" not in output assert "memory_preflight" in output["additionalContext"] @@ -64,46 +86,67 @@ def test_daem0n_project_exits_zero_with_reminder(tmp_path): assert "\n" not in output["additionalContext"] -def test_falls_back_to_claude_project_dir(tmp_path): - (tmp_path / ".daem0nmcp").mkdir() - event = _write_event(tmp_path, "notes.md") - del event["cwd"] - result = _run( - ["-m", "daem0nmcp.claude_hooks.pre_edit"], - event, - env={"CLAUDE_PROJECT_DIR": str(tmp_path)}, - ) +def test_subdirectory_cwd_uses_the_enclosing_project(tmp_path, project): + sub = project / "src" + sub.mkdir() + result = _pre_edit(_write_event(sub, "app.py"), tmp_path) assert result.returncode == 0 - context = json.loads(result.stdout)["hookSpecificOutput"]["additionalContext"] - assert 'relative_file_path="notes.md"' in context + assert 'relative_file_path="src/app.py"' in _context(result) -def test_path_outside_project_and_malformed_input_still_exit_zero(tmp_path): - (tmp_path / "project" / ".daem0nmcp").mkdir(parents=True) - outside = _run( - ["-m", "daem0nmcp.claude_hooks.pre_edit"], - _write_event(tmp_path / "project", str(tmp_path / "elsewhere.py")), - ) +def test_falls_back_to_claude_project_dir(tmp_path, project): + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + event = _write_event(elsewhere, str(project / "notes.md")) + result = _pre_edit(event, tmp_path, env={"CLAUDE_PROJECT_DIR": str(project)}) + assert result.returncode == 0 + assert 'relative_file_path="notes.md"' in _context(result) + + +def test_home_directory_global_state_is_not_a_project(tmp_path): + (tmp_path / ".daem0nmcp" / "edit-bridges").mkdir(parents=True) + result = _pre_edit(_write_event(tmp_path, str(tmp_path / "a.py")), tmp_path) + assert result.returncode == 0 + assert result.stdout == "" + + +def test_path_outside_project_and_malformed_input_still_exit_zero(tmp_path, project): + outside = _pre_edit(_write_event(project, str(tmp_path / "x.py")), tmp_path) assert outside.returncode == 0 - context = json.loads(outside.stdout)["hookSpecificOutput"]["additionalContext"] - assert "relative_file_path" not in context + assert "relative_file_path" not in _context(outside) - environment = {**os.environ, "PYTHONPATH": str(ROOT)} - garbage = subprocess.run( - [sys.executable, "-m", "daem0nmcp.claude_hooks.pre_edit"], - input="not json", - capture_output=True, - text=True, - env=environment, - timeout=30, - check=False, - ) + garbage = _pre_edit("not json", tmp_path) assert garbage.returncode == 0 assert garbage.stdout == "" -def test_imports_no_api_or_retrieval_modules(tmp_path): - (tmp_path / ".daem0nmcp").mkdir() +def test_file_name_is_json_escaped_in_the_reminder(tmp_path, project): + result = _pre_edit(_write_event(project, 'a"b).py'), tmp_path) + assert result.returncode == 0 + assert 'relative_file_path="a\\"b).py"' in _context(result) + + +@pytest.mark.parametrize( + "raw", + [ + r"\\attacker-host\share\x.py", + "//attacker-host/share/x.py", + "../outside.py", + "C:other.py" if os.name == "nt" else "/etc/passwd", + ], +) +def test_relative_path_is_lexical_and_drops_unc_or_outside(tmp_path, monkeypatch, raw): + def no_filesystem(*_args, **_kwargs): + raise AssertionError("model-supplied paths must not touch the filesystem") + + monkeypatch.setattr(Path, "resolve", no_filesystem) + monkeypatch.setattr(os, "stat", no_filesystem) + monkeypatch.setattr(os.path, "realpath", no_filesystem) + assert relative_project_path(tmp_path / "project", raw) is None + assert relative_project_path(tmp_path / "project", "src/../a.py") == "a.py" + + +def test_imports_no_api_or_retrieval_modules(tmp_path, project): probe = ( "import sys, runpy\n" "try:\n" @@ -116,6 +159,6 @@ def test_imports_no_api_or_retrieval_modules(tmp_path): "print(heavy, file=sys.stderr)\n" "sys.exit(1 if heavy else 0)\n" ) - result = _run(["-c", probe], _write_event(tmp_path, str(tmp_path / "a.py"))) + result = _run(["-c", probe], _write_event(project, str(project / "a.py")), tmp_path) assert result.returncode == 0, result.stderr assert "additionalContext" in result.stdout diff --git a/tests/test_claude_hooks_stop.py b/tests/test_claude_hooks_stop.py index 1b585fd..41627eb 100644 --- a/tests/test_claude_hooks_stop.py +++ b/tests/test_claude_hooks_stop.py @@ -71,7 +71,8 @@ async def test_suggests_decisions_without_writing_memory(tmp_project): ] state = {"reminder_count": 0, "last_reminder_turn": -1} result = await analyse_and_remember(str(tmp_project), messages, state) - assert "hook did not write memory" in result.message + assert "The hook wrote nothing" in result.message + assert "ask Claude" in result.message assert "mcp__daem0nmcp__memory_preflight" in result.message assert "mcp__daem0nmcp__memory_store" in result.message @@ -138,8 +139,19 @@ def _run_stop(event: dict, home: Path) -> subprocess.CompletedProcess[str]: def test_main_reads_stdin_event_and_nested_transcript(tmp_path): project = tmp_path / "project" (project / ".daem0nmcp").mkdir(parents=True) + (project / "src").mkdir() + state_dir = tmp_path / ".daem0nmcp" / "hook_state" + state_dir.mkdir(parents=True) + stale = state_dir / "stop_old-session.json" + stale.write_text("{}", encoding="utf-8") + os.utime(stale, (0, 0)) transcript = tmp_path / "transcript.jsonl" + filler = [ + {"type": "user", "message": {"role": "user", "content": f"step {n}"}} + for n in range(300) + ] records = [ + *filler, {"type": "summary", "summary": "Caching work"}, { "type": "user", @@ -169,7 +181,7 @@ def test_main_reads_stdin_event_and_nested_transcript(tmp_path): event = { "session_id": "s-1", "transcript_path": str(transcript), - "cwd": str(project), + "cwd": str(project / "src"), "hook_event_name": "Stop", "stop_hook_active": False, } @@ -182,7 +194,9 @@ def test_main_reads_stdin_event_and_nested_transcript(tmp_path): assert "mcp__daem0nmcp__memory_store" in message assert "memory_record_outcome" in message assert not (project / ".daem0nmcp" / "storage").exists() - assert (tmp_path / ".daem0nmcp" / "hook_state" / "stop_s-1.json").is_file() + state = json.loads((state_dir / "stop_s-1.json").read_text(encoding="utf-8")) + assert state["last_reminder_turn"] == len(records) + assert not stale.exists() def test_main_is_silent_outside_a_daem0n_project(tmp_path): From 27d1682722650932c00331f3b9b12eb75df12906 Mon Sep 17 00:00:00 2001 From: Devil Date: Fri, 18 Sep 2026 23:04:06 -0400 Subject: [PATCH 29/72] docs: note OpenCode edit gating is gone and document project detection Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 4 +++- README.md | 16 ++++++++++------ Summon_Daem0n.md | 4 ++-- docs/v7-operations.md | 10 ++++++---- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c7750..dca386b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,12 @@ ### Added - Reproducible v7 core foundation with opt-in dependency profiles and lazy capability reporting. -- `uninstall-claude-hooks` with `--project-path` strips the pairing env keys from the project's `.claude/settings.local.json`; `--remove-credentials` also deletes its local edit-bridge credential and socket directories. +- `uninstall-claude-hooks` strips the pairing env keys from the project's `.claude/settings.local.json` (the project is `--project-path` or the current directory, and only a project the installer paired is touched); `--remove-credentials` also deletes its local edit-bridge credential and socket directories. ### Changed - Claude Code hooks remind instead of block. `pre_edit` adds a one-line `memory_recall_file`/`memory_preflight` reminder to the model's context and never denies an edit; `stop` reads its stdin event, shows suggestions as a `systemMessage` and never keeps the agent running. +- OpenCode edits are no longer gated either: the OpenCode plugin runs the same `pre_edit` hook, which now always allows the edit. The plugin does not yet show the reminder; that rework comes with the edit-bridge removal. +- `install-claude-hooks` and `uninstall-claude-hooks` refuse to touch a `~/.claude/settings.json` that is not valid JSON, instead of replacing it. - The manual `hooks/daem0n_*.py` scripts are deprecated stubs that print a notice and exit 0, and `hooks/settings.json.example` is removed. If you installed them by hand, re-run `python -m daem0nmcp.cli install-claude-hooks`; it also removes the old entries. ## [6.0.0] - 2026-01-29 diff --git a/README.md b/README.md index 728f2da..c2ca8d7 100644 --- a/README.md +++ b/README.md @@ -616,18 +616,20 @@ python -m daem0nmcp.cli install-claude-hooks This writes six hook entries to the user-level `~/.claude/settings.json`, so they run in every project; each runs `"" -m daem0nmcp.claude_hooks.` -with the interpreter you installed from, and stays silent in directories without -`.daem0nmcp/`. Other hooks in that file are kept, and any old +with the interpreter you installed from. A hook only acts inside a Daem0n +project: the session directory, `CLAUDE_PROJECT_DIR`, or one of their parent +directories contains `.daem0nmcp/` (your home directory does not count, since +Daem0n keeps its own state in `~/.daem0nmcp/`). Other hooks in that file are kept, and any old `hooks/daem0n_*.py` entries are removed. | Event (matcher) | Hook | What it does | Input | Reminds only | |-----------------|------|--------------|-------|--------------| | `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | -| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | | `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | | `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | | `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | -| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | No hook blocks a tool call or keeps the agent running: every hook exits 0 and none returns a `permissionDecision` or `decision`. @@ -642,10 +644,12 @@ returns a `permissionDecision` or `decision`. `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, `DAEM0NMCP_PROJECT_ROOT` and `DAEM0NMCP_STORAGE_PATH` in `/.claude/settings.local.json`. -To remove the hooks, and optionally the project's pairing and credential: +To remove the hooks, the project's pairing env keys (only if the installer +paired it) and, optionally, its credential (the project defaults to the current +directory): ```bash -python -m daem0nmcp.cli --project-path uninstall-claude-hooks [--remove-credentials] +python -m daem0nmcp.cli uninstall-claude-hooks [--project-path ] [--remove-credentials] ``` The old manual scripts in `hooks/` are deprecated stubs that only print a diff --git a/Summon_Daem0n.md b/Summon_Daem0n.md index 379a457..c8a0431 100644 --- a/Summon_Daem0n.md +++ b/Summon_Daem0n.md @@ -180,11 +180,11 @@ them writes Daem0n memory. | Event (matcher) | Hook | What it does | Input | Reminds only | |-----------------|------|--------------|-------|--------------| | `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | -| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | | `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | | `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | | `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | -| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | No hook blocks a tool call or keeps the agent running: every hook exits 0 and none returns a `permissionDecision` or `decision`. diff --git a/docs/v7-operations.md b/docs/v7-operations.md index 16e6b70..c7e8b8e 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -258,16 +258,18 @@ python -m daem0nmcp.cli --project-path /srv/project-a install-claude-hooks ``` The hooks go into the user-level `~/.claude/settings.json` and run in every -project, staying silent where there is no `.daem0nmcp/`. They remind only: +project, staying silent unless the session directory, `CLAUDE_PROJECT_DIR`, or +one of their parents (not your home directory) contains `.daem0nmcp/`. They +remind only: | Event (matcher) | Hook | What it does | Input | Reminds only | |-----------------|------|--------------|-------|--------------| | `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | -| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | In a project with `.daem0nmcp/`, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | | `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | | `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | | `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | -| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | No hook blocks a tool call or keeps the agent running: every hook exits 0 and none returns a `permissionDecision` or `decision`. @@ -281,7 +283,7 @@ adds `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR (with `--remove-credentials`) the credential and socket directories with: ```bash -python -m daem0nmcp.cli --project-path /srv/project-a uninstall-claude-hooks --remove-credentials +python -m daem0nmcp.cli uninstall-claude-hooks --project-path /srv/project-a --remove-credentials ``` Install OpenCode V1 locally: From 987bd00dbea0b4de93683f4565a943010abb4ab9 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 00:54:21 -0400 Subject: [PATCH 30/72] fix: stop preflight guidance spawning git, the cause of its timeouts Under load, memory_preflight overran its 2 s read and then the 15 s tool deadline (DEADLINE_EXCEEDED / INTERNAL_ERROR on CI). Stack captures at the deadline showed every resource worker inside _read_git_changes_sync: each briefing snapshot runs two git subprocesses (on Windows also a system-wide Toolhelp thread walk to resume them), and guidance discards git_changes. read_briefing_snapshot gains include_git_changes; the guidance reader passes False. session_brief still reads git changes. Also: a shared storage lock refused because another process holds it exclusively (bootstrap, migration) is DATABASE_IN_USE, not ACTIVE_V7_UNAVAILABLE; and ACTIVE_V7_UNAVAILABLE, opaque on the wire, is now logged with its cause and correlation ID. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/pinned.py | 8 ++++++++ daem0nmcp/api/v7/production.py | 1 + daem0nmcp/api/v7/resource_repository.py | 13 ++++++++++-- daem0nmcp/api/v7/runtime_services.py | 4 ++++ tests/api_v7/test_error_honesty.py | 21 +++++++++++++++++++ tests/api_v7/test_production.py | 2 ++ tests/api_v7/test_resource_repository.py | 26 ++++++++++++++++++++++++ 7 files changed, 73 insertions(+), 2 deletions(-) diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 01ef10a..df85f14 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -3,6 +3,7 @@ from __future__ import annotations import inspect +import logging import os import re from collections.abc import Awaitable, Callable, Mapping @@ -238,6 +239,13 @@ def _expected_service_failure( if mapped is None: return None code, message, retryable = mapped + if error_code == "ACTIVE_V7_UNAVAILABLE": + # Opaque on the wire like INTERNAL_ERROR, so keep the cause findable. + logging.getLogger(__name__).warning( + "v7 workspace unavailable correlation_id=%s", + response.request_id, + exc_info=error, + ) return response.failure( code, message, diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 498cd4a..c114041 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -605,6 +605,7 @@ async def read( failure_limit=20, rule_limit=50, active_context_limit=1, + include_git_changes=False, ) candidate_records = [ *_public_items(snapshot.failures, RecordSummary), diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index 256a0c5..933131a 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -521,8 +521,13 @@ async def read_briefing_snapshot( failure_limit: int, rule_limit: int = 50, active_context_limit: int = 50, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: - """Read all briefing resources without a pointer-generation gap.""" + """Read all briefing resources without a pointer-generation gap. + + Git changes cost two ``git`` subprocesses per read (plus a system-wide + thread walk on Windows); callers that never show them skip them. + """ if not isinstance(workspace, Workspace): raise ValueError("workspace must be a registered Workspace") @@ -563,6 +568,7 @@ async def read_briefing_snapshot( failure_request=failure_request, rule_request=rule_request, active_request=active_request, + include_git_changes=include_git_changes is True, ) ) @@ -574,6 +580,7 @@ def _read_briefing_snapshot_sync( failure_request: ResourceReadRequest | None, rule_request: ResourceReadRequest, active_request: ResourceReadRequest, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: connection, storage_lock = self._open_connection(workspace) try: @@ -645,7 +652,9 @@ def _read_briefing_snapshot_sync( rules=rules, active_context=active_context, decisions=decisions, - git_changes=self._read_git_changes_sync(workspace), + git_changes=( + self._read_git_changes_sync(workspace) if include_git_changes else [] + ), projection_freshness=projection_freshness, workspace_statistics=workspace_statistics, stale_projection_count=stale_projection_count, diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index dd24caa..4b437e2 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -197,6 +197,10 @@ def locked_current( raise except Exception as exc: lock.release() + # DatabaseInUseError: another process holds the storage lock + # exclusively (fresh bootstrap, migration) - transient, retryable. + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc try: yield active diff --git a/tests/api_v7/test_error_honesty.py b/tests/api_v7/test_error_honesty.py index 0c1f407..d248f3b 100644 --- a/tests/api_v7/test_error_honesty.py +++ b/tests/api_v7/test_error_honesty.py @@ -232,6 +232,27 @@ async def pool_occupied() -> None: assert caught.value.code == "DATABASE_IN_USE" +async def test_exclusive_storage_lock_is_database_in_use_not_unavailable(tmp_path): + """A migration/bootstrap holding the storage lock is transient contention.""" + from daem0nmcp.api.v7.runtime_services import WorkspaceStorageResolver + from daem0nmcp.storage_activation import DatabaseFileLock + + [workspace] = await initialize_workspaces((tmp_path,)) + holder = DatabaseFileLock(tmp_path / ".daem0nmcp" / "storage", "exclusive") + holder.acquire() + try: + with ( + pytest.raises(RuntimeServiceError) as caught, + WorkspaceStorageResolver().locked_active(workspace), + ): + pytest.fail("a shared read was admitted under an exclusive lock") + finally: + holder.release() + assert caught.value.code == "DATABASE_IN_USE" + with WorkspaceStorageResolver().locked_active(workspace) as active: + assert active.format_version == 7 + + def test_internal_error_log_omits_pydantic_input_values(caplog): from pydantic import BaseModel, ValidationError diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index c6f1ac2..1c71d37 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -409,6 +409,8 @@ async def snapshot(_workspace, **limits): ) self.assertEqual(len(calls), 1) + # Guidance never shows git changes, so it must not spawn git for them. + self.assertIs(calls[0]["include_git_changes"], False) self.assertEqual(result["records"], [relevant_failure]) self.assertEqual(result["rules"], [relevant_rule]) self.assertIn( diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index 76d1e95..f60ff9d 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -838,6 +838,32 @@ def read_records(*args, **kwargs): ) self.assertEqual(snapshot.workspace_statistics["active_context"], 2) + async def test_snapshot_skips_git_subprocesses_when_changes_are_unused( + self, + ) -> None: + # Preflight guidance paid two git subprocesses per call for data it + # discarded; under load that pushed reads past their deadline. + from daem0nmcp.api.v7.resource_repository import SQLiteResourceRepository + + repository = SQLiteResourceRepository( + lambda _workspace: self.fixture.resolved, + clock=lambda: NOW, + ) + git_reads: list[object] = [] + + def read_git_changes(workspace): + git_reads.append(workspace) + return [] + + repository._read_git_changes_sync = read_git_changes + limits = {"warning_limit": 1, "failure_limit": 1, "active_context_limit": 1} + skipped = await repository.read_briefing_snapshot( + self.fixture.workspace, include_git_changes=False, **limits + ) + self.assertEqual((skipped.git_changes, git_reads), ([], [])) + await repository.read_briefing_snapshot(self.fixture.workspace, **limits) + self.assertEqual(len(git_reads), 1) + async def test_briefing_sections_share_one_sqlite_read_snapshot(self) -> None: # A canonical writer may commit while holding the same shared generation # lock. Every section of one brief must still observe one SQLite snapshot. From 6eb091e7bd0671e788c06da1876d2d2239baa681 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 01:22:41 -0400 Subject: [PATCH 31/72] feat: enable an optional profile automatically when its extra is installed An unset DAEM0NMCP__ENABLED now means auto: ready when every distribution is installed, disabled (with install remediation) otherwise. An explicit false still disables; an explicit true with packages missing stays degraded. Fix the apps and index remediation text to match. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/discovery_operations.py | 5 +- daem0nmcp/capabilities.py | 39 +++++++++++--- daem0nmcp/cli.py | 9 ++-- tests/test_capabilities.py | 69 ++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 10 deletions(-) diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..d9a9a6d 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -1424,7 +1424,10 @@ def _code_capability_error( else "CAPABILITY_DEGRADED" ) remediation = ( - "Set DAEM0NMCP_APPS_ENABLED=true and install the apps dependency profile." + "Set DAEM0NMCP_APPS_ENABLED to true, false, or leave it unset." + if capability_status == "failed" + else "Install the apps profile with pip install 'daem0nmcp[apps]'; " + "it turns on automatically unless DAEM0NMCP_APPS_ENABLED=false." ) return DiscoveryOperationError( code, diff --git a/daem0nmcp/capabilities.py b/daem0nmcp/capabilities.py index 6b17e0e..36f022b 100644 --- a/daem0nmcp/capabilities.py +++ b/daem0nmcp/capabilities.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from importlib import metadata -from typing import Any +from typing import Any, Literal CAPABILITY_STATUSES = frozenset({"ready", "disabled", "degraded", "failed"}) CORE_DISTRIBUTIONS = ( @@ -117,10 +117,15 @@ def get(self, name: str) -> dict[str, Any]: configured = self._configured(profile) if configured is None: return self._failed(profile) - if not configured: + if configured is False: return self._disabled(profile) + # An unset variable means "on when installed": a profile that cannot run + # is simply off, while an explicit request for it is degraded. if name == "models-local" and sys.version_info < (3, 11): - return self._python_version_unavailable(profile) + capability = self._python_version_unavailable(profile) + if configured == "auto": + capability["status"] = "disabled" + return capability missing = [ distribution @@ -128,6 +133,8 @@ def get(self, name: str) -> dict[str, Any]: if not self._distribution_available(distribution) ] if missing: + if configured == "auto": + return self._not_installed(profile, missing) return self._degraded(profile, missing) return self._ready(profile) @@ -167,11 +174,13 @@ def _core(self) -> dict[str, Any]: }, } - def _configured(self, profile: CapabilityProfile) -> bool | None: - value = self._environ.get(profile.environment_key, "false").strip().lower() + def _configured(self, profile: CapabilityProfile) -> bool | Literal["auto"] | None: + value = self._environ.get(profile.environment_key, "").strip().lower() + if not value: + return "auto" if value in {"1", "true", "yes", "on"}: return True - if value in {"0", "false", "no", "off", ""}: + if value in {"0", "false", "no", "off"}: return False return None @@ -191,6 +200,24 @@ def _disabled(profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "enable_configuration", "environment": f"{profile.environment_key}=true", + "message": ( + f"Unset {profile.environment_key} or set it to true; the " + f"profile turns on when 'daem0nmcp[{profile.name}]' is installed." + ), + }, + } + + @staticmethod + def _not_installed( + profile: CapabilityProfile, missing: list[str] + ) -> dict[str, Any]: + return { + "name": profile.name, + "status": "disabled", + "remediation": { + "action": "install_extra", + "command": f"pip install 'daem0nmcp[{profile.name}]'", + "missing": missing, }, } diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..31e065b 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -1218,15 +1218,18 @@ async def run_watcher(): if args.json: print( json.dumps( - {"error": "tree-sitter-languages not installed", "indexed": 0} + { + "error": "tree-sitter-language-pack not installed", + "indexed": 0, + } ) ) else: print( - "ERROR: Code indexing requires tree-sitter-languages", + "ERROR: Code indexing requires tree-sitter-language-pack", file=sys.stderr, ) - print("Install with: pip install tree-sitter-languages") + print("Install with: pip install 'daem0nmcp[apps]'") sys.exit(1) project_path = Path(args.project_path or os.getcwd()).resolve() diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 8459d84..dfeb94a 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -218,6 +218,75 @@ def test_disabled_capability_does_not_probe_its_optional_modules(self): "DAEM0NMCP_MODELS_LOCAL_ENABLED=true", ) + def test_unset_variable_enables_an_installed_profile(self): + """An installed extra turns its profile on without any variable.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("graph") + + self.assertEqual(capability["status"], "ready") + + def test_unset_variable_leaves_a_missing_extra_disabled(self): + """Without the extra, an unrequested profile is off, not degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda name: name != "leidenalg" + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual( + capability["remediation"], + { + "action": "install_extra", + "command": "pip install 'daem0nmcp[graph]'", + "missing": ["leidenalg"], + }, + ) + + def test_explicit_false_disables_an_installed_profile(self): + """The variable still turns an installed profile off.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + module_available=lambda _: True, + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "enable_configuration") + self.assertIn( + "Unset DAEM0NMCP_GRAPH_ENABLED or set it to true", + capability["remediation"]["message"], + ) + + def test_explicit_true_with_a_missing_extra_is_degraded(self): + """Asking for a profile whose packages are absent is a degraded request.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "true"}, + module_available=lambda name: name != "leidenalg", + ).get("graph") + + self.assertEqual(capability["status"], "degraded") + self.assertEqual(capability["remediation"]["action"], "install_extra") + self.assertEqual(capability["remediation"]["missing"], ["leidenalg"]) + + def test_unset_models_local_is_disabled_on_unsupported_python(self): + """Auto mode never reports an unrequested profile as degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + with patch("daem0nmcp.capabilities.sys.version_info", (3, 10, 21)): + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("models-local") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "upgrade_python") + def test_invalid_enablement_value_is_failed_with_structured_remediation(self): """An invalid profile setting is a configuration failure, not a missing extra.""" from daem0nmcp.capabilities import CapabilityRegistry From e63f1e08d49f9cd5642d5b7a337d48cbe1db860f Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 01:22:48 -0400 Subject: [PATCH 32/72] fix: make the Windows launcher serve the caller's project start_server.py now uses --project, else an exported DAEM0NMCP_PROJECT_ROOT, else the current directory, and never overwrites an exported value. The .bat captures %CD% (or takes %1) before changing to the repo directory and passes it as --project. Co-Authored-By: Claude Opus 5 --- start_daem0nmcp_server.bat | 16 ++++++-- start_server.py | 11 +++-- tests/test_start_server.py | 82 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 tests/test_start_server.py diff --git a/start_daem0nmcp_server.bat b/start_daem0nmcp_server.bat index fb71a2c..e2a0704 100644 --- a/start_daem0nmcp_server.bat +++ b/start_daem0nmcp_server.bat @@ -3,13 +3,20 @@ REM ============================================ REM Daem0nMCP HTTP Server Launcher for Windows REM ============================================ REM This script starts the Daem0nMCP HTTP server -REM for use with Claude Code on Windows. +REM for one project on Windows. REM -REM Windows has known issues with stdio transport, -REM so HTTP transport is required. +REM Usage: start_daem0nmcp_server.bat [project_dir] +REM The project defaults to the directory you run +REM it from. Stdio (python -m daem0nmcp) also works +REM on Windows; use this launcher when you want an +REM HTTP server instead. REM ============================================ title Daem0nMCP Server +setlocal + +REM Capture the project before changing directory +if "%~1"=="" (set "DAEM0N_PROJECT=%CD%") else (set "DAEM0N_PROJECT=%~f1") REM Change to the script's directory cd /d "%~dp0" @@ -26,7 +33,8 @@ echo * echo. REM Start the server -python start_server.py --port 9876 +REM The trailing "\." keeps a drive root such as C:\ from escaping the quote +python start_server.py --port 9876 --project "%DAEM0N_PROJECT%\." REM If the server exits, pause so user can see any errors if errorlevel 1 ( diff --git a/start_server.py b/start_server.py index ef3ffb7..54cbcc7 100644 --- a/start_server.py +++ b/start_server.py @@ -22,12 +22,17 @@ def main(): parser.add_argument("--host", default="127.0.0.1", help="Host to bind to") parser.add_argument( "--project", - help="Project directory for storage (default: current directory)", + help=( + "Project directory to serve (default: DAEM0NMCP_PROJECT_ROOT if set, " + "else the current directory)" + ), ) args = parser.parse_args() - project_root = Path(args.project or os.getcwd()).resolve() - os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root) + if args.project: + os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(Path(args.project).resolve()) + elif not os.environ.get("DAEM0NMCP_PROJECT_ROOT"): + os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(Path.cwd().resolve()) # Import only after the workspace environment is fixed. Both public # launchers use the same v7 composition and transport-security boundary. diff --git a/tests/test_start_server.py b/tests/test_start_server.py new file mode 100644 index 0000000..3629942 --- /dev/null +++ b/tests/test_start_server.py @@ -0,0 +1,82 @@ +"""The HTTP launchers serve the user's project, never the Daem0n-MCP checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _launched_project_root(argv: list[str], environ: dict[str, str]) -> str: + import start_server + + seen: list[str] = [] + + def create_server(transport: str, *, host: str | None = None) -> object: + seen.append(os.environ["DAEM0NMCP_PROJECT_ROOT"]) + return object() + + fake_server = types.ModuleType("daem0nmcp.server") + fake_server.create_server = create_server + with ( + patch.dict(sys.modules, {"daem0nmcp.server": fake_server}), + patch.object(sys, "argv", ["start_server.py", *argv]), + patch.dict(os.environ, environ, clear=True), + patch("daem0nmcp.api.v7.launcher.run_server"), + ): + start_server.main() + return seen[0] + + +def test_exported_project_root_survives(tmp_path): + exported = str(tmp_path / "exported") + assert _launched_project_root([], {"DAEM0NMCP_PROJECT_ROOT": exported}) == exported + + +def test_project_argument_wins_over_exported_root(tmp_path): + chosen = tmp_path / "chosen" + root = _launched_project_root( + ["--project", str(chosen)], + {"DAEM0NMCP_PROJECT_ROOT": str(tmp_path / "exported")}, + ) + assert root == str(chosen.resolve()) + + +def test_current_directory_is_the_default(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert _launched_project_root([], {}) == str(tmp_path.resolve()) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +@pytest.mark.parametrize("use_argument", [False, True]) +def test_batch_launcher_passes_the_callers_project(tmp_path, use_argument): + """The .bat changes to the repo dir, so it must capture the project first.""" + shim = tmp_path / "shim" + shim.mkdir() + recorded = tmp_path / "argv.txt" + (shim / "python.cmd").write_text(f'@echo %*> "{recorded}"\r\n', encoding="ascii") + project = tmp_path / "my project (x86)" + project.mkdir() + environment = {**os.environ, "PATH": f"{shim}{os.pathsep}{os.environ['PATH']}"} + + subprocess.run( + ["cmd", "/c", str(REPO_ROOT / "start_daem0nmcp_server.bat")] + + (["my project (x86)"] if use_argument else []), + cwd=tmp_path if use_argument else project, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + check=True, + timeout=30, + ) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == f'start_server.py --port 9876 --project "{project}\\."' + assert Path(f"{project}\\.").resolve() == project.resolve() From f2e6a5dddc473d90f02274f6338ceef120f1b25b Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 01:22:48 -0400 Subject: [PATCH 33/72] fix: install the OpenCode slash-command files The installer copies the four repo command files (packaged under opencode_assets/commands) into .opencode/commands/, creating only the ones that are absent unless --force is given. Co-Authored-By: Claude Opus 5 --- daem0nmcp/opencode_assets/commands/commune.md | 13 ++++++++ daem0nmcp/opencode_assets/commands/counsel.md | 24 ++++++++++++++ .../opencode_assets/commands/inscribe.md | 31 +++++++++++++++++++ daem0nmcp/opencode_assets/commands/recall.md | 14 +++++++++ daem0nmcp/opencode_install.py | 27 ++++++++++------ pyproject.toml | 2 +- tests/test_opencode_install.py | 24 ++++++++++++++ 7 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 daem0nmcp/opencode_assets/commands/commune.md create mode 100644 daem0nmcp/opencode_assets/commands/counsel.md create mode 100644 daem0nmcp/opencode_assets/commands/inscribe.md create mode 100644 daem0nmcp/opencode_assets/commands/recall.md diff --git a/daem0nmcp/opencode_assets/commands/commune.md b/daem0nmcp/opencode_assets/commands/commune.md new file mode 100644 index 0000000..27a2781 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/commune.md @@ -0,0 +1,13 @@ +--- +description: Start the Daem0n v7 session for a registered workspace +--- + +Call `daem0nmcp_session_brief(workspace_id="$ARGUMENTS")` immediately. The +argument must be the configured opaque `workspace_id`, not a path. + +After receiving the briefing results: +1. Report the session status, active warnings, and recent activity summary +2. Acknowledge that the v7 scoped session is active + +This slash-command filename is only a host shortcut. For v6 migration details, +use `docs/v6-to-v7-tools.json`. diff --git a/daem0nmcp/opencode_assets/commands/counsel.md b/daem0nmcp/opencode_assets/commands/counsel.md new file mode 100644 index 0000000..d8cd22f --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/counsel.md @@ -0,0 +1,24 @@ +--- +description: Request an exact Daem0n v7 preflight capability +--- + +Parse `$ARGUMENTS` as the intended protected v7 tool and its arguments, then +call: + +```text +daem0nmcp_memory_preflight( + workspace_id="", + target_tool="", + target_arguments={}, + description="$ARGUMENTS" +) +``` + +After receiving the preflight results: +1. Report any WARNINGS the user must know about +2. Report any FAILED APPROACHES that are relevant +3. Report any must_not constraints that apply +4. Preserve the token only for the exact request it authorizes + +Do not claim a generic edit authorization. The server binds the capability to +the workspace, principal, session, target tool, and canonical arguments. diff --git a/daem0nmcp/opencode_assets/commands/inscribe.md b/daem0nmcp/opencode_assets/commands/inscribe.md new file mode 100644 index 0000000..1359bb2 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/inscribe.md @@ -0,0 +1,31 @@ +--- +description: Store a replay-safe Daem0n v7 memory record +--- + +Store the following as a v7 memory record: + +$ARGUMENTS + +Choose `record_type` from the content: +- "decision" for architectural or design choices +- "pattern" for recurring approaches to follow +- "warning" for things to avoid +- "learning" for lessons from experience + +Create one stable idempotency key. Call `daem0nmcp_memory_preflight` with +`target_tool="memory_store"` and the exact target arguments, then call: + +```text +daem0nmcp_memory_store( + workspace_id="", + record_type="", + content="$ARGUMENTS", + rationale="", + idempotency_key="", + preflight_token="" +) +``` + +Report the returned `record_id`. After verification, use +`daem0nmcp_memory_record_outcome` with that ID and a separate stable +idempotency key. diff --git a/daem0nmcp/opencode_assets/commands/recall.md b/daem0nmcp/opencode_assets/commands/recall.md new file mode 100644 index 0000000..e372a59 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/recall.md @@ -0,0 +1,14 @@ +--- +description: Retrieve bounded memories from a Daem0n v7 workspace +--- + +Call `daem0nmcp_memory_recall(workspace_id="", query="$ARGUMENTS", limit=10)` immediately. + +Present the retrieved memories in a clear format: +- Show each record's content, type, and creation time +- Highlight any warnings or failed approaches +- Preserve evidence references and truncation indicators + +For service diagnostics use +`daem0nmcp_system_health(workspace_id="")`. Do not replace the +opaque workspace ID with a project path. diff --git a/daem0nmcp/opencode_install.py b/daem0nmcp/opencode_install.py index bddb83a..36d9af4 100644 --- a/daem0nmcp/opencode_install.py +++ b/daem0nmcp/opencode_install.py @@ -1,8 +1,8 @@ """Installer for OpenCode integration. -Creates .opencode/ directory structure, ensures opencode.json exists -at project root, and writes the TypeScript covenant enforcement plugin -for MCP server connectivity and hook discipline. +Creates the .opencode/ directory structure and slash-command files, ensures +opencode.json exists at project root, and writes the TypeScript covenant +enforcement plugin for MCP server connectivity and hook discipline. """ import argparse @@ -44,12 +44,14 @@ def select_opencode_interface(version: str, requested: str | None = None) -> str # Subdirectories to scaffold inside .opencode/ _OPENCODE_SUBDIRS = ["commands", "plugins", "agents"] -# The packaged TypeScript resource is the sole installer source of truth. -PLUGIN_TEMPLATE = ( - importlib.resources.files("daem0nmcp.opencode_assets") - .joinpath("daem0n.ts") - .read_text(encoding="utf-8") -) +# The packaged resources are the sole installer source of truth. +_ASSETS = importlib.resources.files("daem0nmcp.opencode_assets") +PLUGIN_TEMPLATE = _ASSETS.joinpath("daem0n.ts").read_text(encoding="utf-8") +COMMAND_TEMPLATES = { + command.name: command.read_text(encoding="utf-8") + for command in _ASSETS.joinpath("commands").iterdir() + if command.name.endswith(".md") +} def detect_clients(project_path: Path) -> dict[str, Any]: @@ -297,6 +299,13 @@ def install_opencode( status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force) lines.append(f" {status} .opencode/plugins/daem0n.ts") + # -- Ensure slash-command files ---------------------------------- + for name, content in sorted(COMMAND_TEMPLATES.items()): + status = _ensure_file( + opencode_root / "commands" / name, content, dry_run, force + ) + lines.append(f" {status} .opencode/commands/{name}") + # -- AGENTS.md status (do NOT create) ---------------------------- if oc["agents_md"]: lines.append(" [exists] AGENTS.md") diff --git a/pyproject.toml b/pyproject.toml index 09a50ed..e906eb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ include = ["daem0nmcp*"] "static/*.js", "static/renderers/*.js", ] -"daem0nmcp.opencode_assets" = ["*.ts"] +"daem0nmcp.opencode_assets" = ["*.ts", "commands/*.md"] # Ruff configuration [tool.ruff] diff --git a/tests/test_opencode_install.py b/tests/test_opencode_install.py index a94b63a..d9d7e50 100644 --- a/tests/test_opencode_install.py +++ b/tests/test_opencode_install.py @@ -228,3 +228,27 @@ def test_install_preserves_existing_opencode_json(tmp_path): assert plugin.exists(), ( "Plugin file should be created even when opencode.json exists" ) + + +def test_install_creates_repo_command_files_without_overwriting(tmp_path): + """The installer ships the repo's slash commands and keeps user edits.""" + repo_commands = Path(__file__).resolve().parents[1] / ".opencode" / "commands" + expected = { + path.name: path.read_text(encoding="utf-8") + for path in repo_commands.glob("*.md") + } + assert set(expected) == {"commune.md", "counsel.md", "inscribe.md", "recall.md"} + commands = tmp_path / ".opencode" / "commands" + commands.mkdir(parents=True) + (commands / "recall.md").write_text("my recall", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[exists] .opencode/commands/recall.md" in msg + assert "[create] .opencode/commands/commune.md" in msg + assert (commands / "recall.md").read_text(encoding="utf-8") == "my recall" + for name in ("commune.md", "counsel.md", "inscribe.md"): + assert (commands / name).read_text(encoding="utf-8") == expected[name] From 0a204d39a6355b4d2beda667e963f82bc8f7a251 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 01:47:51 -0400 Subject: [PATCH 34/72] fix: address PR 5 review round 1 - ErrorRemedy.arguments is user JSON: token-error remedies echo the caller's arguments, so a path in content no longer breaks the error (R-1). - Stored user text reads back with any characters it was stored with: drop _sanitized from Tag and the excerpt fields and the control-char refusals on content in intelligence and entity-evolution reads (R-20). - Sandbox sanitized_logs repeat stdout/stderr, so they are user text up to their real 4096-character length (R-2). - Vector payloads stay guarded; export envelopes are guarded outside their data; legacy rows get the events' structured-key check, which now validates relative-path keys with RelativePath (R-3, R-4). - RelativePath values skip the prose substring scan; their own validator is exact (R-21). - legacy_compat keeps path-mentioning context (R-6); delete the dead _safe_code_path (R-5). Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/code_entity_operations.py | 8 +- daem0nmcp/api/v7/discovery_operations.py | 18 +- daem0nmcp/api/v7/intelligence_operations.py | 4 +- daem0nmcp/api/v7/models.py | 46 +- daem0nmcp/api/v7/operations.py | 19 +- daem0nmcp/api/v7/tools.py | 18 +- daem0nmcp/retrieval/legacy_compat.py | 6 +- tests/api_v7/test_models.py | 16 +- tests/api_v7/test_path_policy.py | 417 ++++++++++++++---- tests/api_v7/test_pinned_handlers.py | 9 +- .../test_process_workspace_isolation_sweep.py | 27 +- 11 files changed, 445 insertions(+), 143 deletions(-) diff --git a/daem0nmcp/api/v7/code_entity_operations.py b/daem0nmcp/api/v7/code_entity_operations.py index 492082e..3d45aed 100644 --- a/daem0nmcp/api/v7/code_entity_operations.py +++ b/daem0nmcp/api/v7/code_entity_operations.py @@ -78,7 +78,6 @@ _MAX_ENTITY_EVENTS = 200 _MAX_IMPACT_ENTITIES = 500 _MAX_IMPACT_EDGES = 5_000 -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _EVENT_TYPE_RE = re.compile(r"^[a-z][a-z0-9_.-]{2,79}$") _EVENT_COLUMNS = ( "event_id,workspace_id,stream_id,stream_kind,stream_version,event_type," @@ -934,11 +933,8 @@ def _entity_row( def _safe_event_content(state: Mapping[str, Any]) -> str: content = state.get("content") - if ( - not isinstance(content, str) - or not content - or _CONTROL_RE.search(content) is not None - ): + # Stored user text reads back as stored (UD-3). + if not isinstance(content, str) or not content: raise CodeEntityOperationError("CAPABILITY_DEGRADED") return content diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 563e0a0..009e4d2 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -50,13 +50,7 @@ ) from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import ( - CapabilityState, - Page, - RecordSummary, - RetrievalData, - _relative_path, -) +from .models import CapabilityState, Page, RecordSummary, RetrievalData from .public_ids import ( PublicObjectIdNotFound, PublicObjectIdRepository, @@ -760,16 +754,6 @@ def _partition_metadata( return row_count, content_hash -def _safe_code_path(value: object) -> bool: - if not isinstance(value, str) or not 1 <= len(value) <= 1024 or value == ".": - return False - try: - _relative_path(value) - except ValueError: - return False - return True - - def _validate_entity_partition( connection: sqlite3.Connection, workspace_id: str, diff --git a/daem0nmcp/api/v7/intelligence_operations.py b/daem0nmcp/api/v7/intelligence_operations.py index 773e496..c94d358 100644 --- a/daem0nmcp/api/v7/intelligence_operations.py +++ b/daem0nmcp/api/v7/intelligence_operations.py @@ -62,7 +62,6 @@ _MAX_COMMUNITY_MEMBERS = 20_000 _CLAIM_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|[\r\n]+") _WORD_RE = re.compile(r"[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*") -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _NEGATIONS = frozenset({"no", "not", "never", "none", "without"}) _STOP_WORDS = frozenset( { @@ -368,7 +367,8 @@ def _verified_event(row: sqlite3.Row) -> dict[str, Any]: def _safe_text(value: object) -> str: - if not isinstance(value, str) or not value or _CONTROL_RE.search(value) is not None: + # Stored user text reads back as stored (UD-3). + if not isinstance(value, str) or not value: raise IntelligenceOperationError("CAPABILITY_DEGRADED") return value diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index 3eefd62..d655b71 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -72,16 +72,22 @@ def is_host_absolute_path(value: str) -> bool: ) -class _UserTextMarker: - """``Annotated`` marker for user-authored or user-derived text.""" +class _Exempt: + """``Annotated`` marker for strings the substring path scan skips.""" - __slots__ = () + __slots__ = ("name",) + + def __init__(self, name: str) -> None: + self.name = name def __repr__(self) -> str: - return "UserText" + return self.name -_USER_TEXT = _UserTextMarker() +_USER_TEXT = _Exempt("UserText") +# RelativePath's own validator refuses absolute and escaping paths exactly; +# the prose scan would also refuse valid names such as ``src/__pycache__/x``. +_RELATIVE_PATH = _Exempt("RelativePath") # Memory content, queries, rule text and server text quoting them may mention # routes and file paths ("/health", "C:\proj\app.py"). Fields typed with it, @@ -90,21 +96,30 @@ def __repr__(self) -> str: UserText = Annotated[str, _USER_TEXT] -def _carries_user_text(annotation: object) -> bool: - return annotation is _USER_TEXT or any( - _carries_user_text(argument) for argument in get_args(annotation) +def _carries(annotation: object, marker: _Exempt) -> bool: + return annotation is marker or any( + _carries(argument, marker) for argument in get_args(annotation) ) -@cache -def _user_text_fields(model: type[BaseModel]) -> frozenset[str]: +def _fields_marked(model: type[BaseModel], marker: _Exempt) -> frozenset[str]: return frozenset( name for name, field in model.model_fields.items() - if _USER_TEXT in field.metadata or _carries_user_text(field.annotation) + if marker in field.metadata or _carries(field.annotation, marker) ) +@cache +def _user_text_fields(model: type[BaseModel]) -> frozenset[str]: + return _fields_marked(model, _USER_TEXT) + + +@cache +def _exempt_fields(model: type[BaseModel]) -> frozenset[str]: + return _user_text_fields(model) | _fields_marked(model, _RELATIVE_PATH) + + def guarded_strings(value: object, user_text: bool = False) -> Iterator[str]: """Yield every string, keys included, that the absolute-path rule covers. @@ -113,7 +128,7 @@ def guarded_strings(value: object, user_text: bool = False) -> Iterator[str]: """ if isinstance(value, BaseModel): - exempt = _user_text_fields(type(value)) + exempt = _exempt_fields(type(value)) for name, item in value.__dict__.items(): yield from guarded_strings(item, name in exempt) elif isinstance(value, Mapping): @@ -339,6 +354,7 @@ def _document_validation_error( str, StringConstraints(strict=True, min_length=1, max_length=1024), AfterValidator(_relative_path), + _RELATIVE_PATH, ] AwareDateTime = Annotated[datetime, BeforeValidator(parse_wire_datetime)] UtcDateTime = Annotated[AwareDateTime, AfterValidator(_utc)] @@ -407,7 +423,6 @@ def _document_validation_error( Tag = Annotated[ UserText, StringConstraints(strict=True, min_length=1, max_length=80), - AfterValidator(_sanitized), ] ProviderName = Annotated[ str, @@ -528,7 +543,8 @@ class FieldError(WireModel): class ErrorRemedy(WireModel): tool: ToolName - arguments: JsonObject = Field(default_factory=dict) + # The caller's own validated arguments, echoed back for the retry. + arguments: UserJsonObject = Field(default_factory=dict) Remedy = ErrorRemedy @@ -645,7 +661,6 @@ class RecordSummary(WireModel): excerpt: Annotated[ UserText, StringConstraints(strict=True, min_length=1, max_length=4000), - AfterValidator(_sanitized), ] tags: Annotated[list[Tag], AfterValidator(_unique_strings)] = Field( default_factory=list, @@ -680,7 +695,6 @@ class EvidenceItem(WireModel): bounded_excerpt: Annotated[ UserText, StringConstraints(strict=True, min_length=1, max_length=8000), - AfterValidator(_sanitized), ] channels: Annotated[list[ProviderName], AfterValidator(_unique_strings)] = Field( min_length=1, diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index 90ef0f1..ff1e6b4 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -24,6 +24,8 @@ from types import MappingProxyType from typing import Any +from pydantic import TypeAdapter, ValidationError + from ...bounded_workers import BoundedWorkerBusyError, BoundedWorkerPool from ...covenant import CovenantGate, InvocationScope from ...event_store import ( @@ -51,7 +53,7 @@ from ...workspace import Workspace from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import EvidenceRef, Page, RecordSummary, is_host_absolute_path +from .models import EvidenceRef, Page, RecordSummary, RelativePath from .portable_projections import ( ImportFinalizationLease, PortableTransferError, @@ -92,6 +94,7 @@ # Host-path columns that are always empty in v7 payloads. _RAW_PATH_KEYS = frozenset({"file_path", "project_path", "database_path"}) _RELATIVE_PATH_KEYS = frozenset({"relative_file_path", "file_path_relative"}) +_RELATIVE_PATH_ADAPTER: TypeAdapter[str] = TypeAdapter(RelativePath) _EVENT_COLUMNS = ( "event_id,workspace_id,stream_id,stream_kind,stream_version,event_type," "event_schema_version,occurred_at_us,recorded_at_us,actor_type,actor_id," @@ -457,12 +460,11 @@ def _reject_raw_paths(value: object) -> None: for key, item in value.items(): if key in _RAW_PATH_KEYS and item is not None and item != "": raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - if ( - key in _RELATIVE_PATH_KEYS - and isinstance(item, str) - and is_host_absolute_path(item) - ): - raise CoreOperationError("WORKSPACE_PATH_ESCAPE") + if key in _RELATIVE_PATH_KEYS and item is not None: + try: + _RELATIVE_PATH_ADAPTER.validate_python(item) + except ValidationError: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") from None if key != "context": _reject_raw_paths(item) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): @@ -996,6 +998,8 @@ def _export_sync( content = page["content"] manifest = page["manifest"] items = content["items"] + if page["page_kind"] == "legacy": + _reject_raw_paths(items) descriptor = page.get( "page_descriptor", { @@ -1054,6 +1058,7 @@ def _portable_page(bundle: ExportBundle) -> dict[str, Any]: items = [event.model_dump(mode="json") for event in bundle.events] elif bundle.page_kind == "legacy": items = list(bundle.legacy_rows) + _reject_raw_paths(items) else: items = [point.model_dump(mode="json") for point in bundle.vector_points] return { diff --git a/daem0nmcp/api/v7/tools.py b/daem0nmcp/api/v7/tools.py index 3332b86..b1fa7c9 100644 --- a/daem0nmcp/api/v7/tools.py +++ b/daem0nmcp/api/v7/tools.py @@ -59,6 +59,7 @@ VersionId, WireModel, WorkspaceId, + contains_absolute_filesystem_path, ) from .policy import V7_TOOL_LEVELS, V7ArgumentNormalizer from .registry import PINNED_TOOL_NAMES, ManifestError, ToolSpec @@ -676,7 +677,12 @@ class SandboxExecutionData(WireModel): stderr: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] exit_status: Annotated[int, Field(ge=-1, le=255)] execution_time_ms: Annotated[int, Field(ge=0, le=60_000)] - sanitized_logs: list[MediumText] = Field(default_factory=list, max_length=100) + # The same lines as stdout/stderr (see external_operations), so user text. + sanitized_logs: list[ + Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=4096) + ] + ] = Field(default_factory=list, max_length=100) class CodeIndexData(WireModel): @@ -869,8 +875,16 @@ class ExportEvent(WireModel): event_type: NameText happened_at: UtcDateTime content_hash: ContentHash + # Only the envelope's ``data`` holds user text; the rest stays guarded. payload: UserJsonObject + @model_validator(mode="after") + def guard_envelope(self) -> ExportEvent: + envelope = {k: v for k, v in self.payload.items() if k != "data"} + if contains_absolute_filesystem_path(envelope): + raise ValueError("absolute filesystem paths are forbidden on the v7 wire") + return self + TransferSessionId = Annotated[ str, @@ -887,7 +901,7 @@ class PortableVectorPoint(WireModel): point_id: Annotated[ str, StringConstraints(strict=True, min_length=1, max_length=256) ] - payload: UserJsonObject + payload: JsonObject vector: list[Annotated[float, Field(allow_inf_nan=False)]] = Field( min_length=1, max_length=4096 ) diff --git a/daem0nmcp/retrieval/legacy_compat.py b/daem0nmcp/retrieval/legacy_compat.py index dc405c6..4a80ac9 100644 --- a/daem0nmcp/retrieval/legacy_compat.py +++ b/daem0nmcp/retrieval/legacy_compat.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Any -from ..api.v7.models import contains_absolute_filesystem_path from ..event_store import ( event_hash_for, event_id_for_hash, @@ -57,9 +56,8 @@ def _safe_legacy_context(value: object) -> dict[str, Any] | None: ).encode("utf-8") except (TypeError, ValueError, RecursionError): return None - if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES or contains_absolute_filesystem_path( - value - ): + # Context is user text and may mention paths (UD-3). + if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES: return None # Detach the value from the parsed event payload before it reaches a cache. copied = json.loads(encoded.decode("utf-8")) diff --git a/tests/api_v7/test_models.py b/tests/api_v7/test_models.py index d461679..511e0fa 100644 --- a/tests/api_v7/test_models.py +++ b/tests/api_v7/test_models.py @@ -250,14 +250,16 @@ def test_json_values_are_strict_finite_bounded_and_duplicate_safe(self) -> None: def test_wire_models_reject_absolute_paths_outside_user_text(self) -> None: _, models = _load(self) + # A remedy echoes the caller's own arguments, which may mention paths. + models.ErrorRemedy( + tool="memory_store", + arguments={"context": {"note": "Inspect C:\\private\\secret.txt"}}, + ) with self.assertRaises(ValidationError): - models.ErrorRemedy( - tool="memory_store", - arguments={ - "context": { - "note": "Inspect C:\\private\\secret.txt before release" - } - }, + models.CapabilityState( + name="storage", + status="failed", + remediation="Repair C:\\private\\storage", ) summary = { diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py index abf570d..b5b9f61 100644 --- a/tests/api_v7/test_path_policy.py +++ b/tests/api_v7/test_path_policy.py @@ -21,12 +21,14 @@ from fastmcp import Client from pydantic import BaseModel, ValidationError, create_model +from daem0nmcp.api.v7.errors import ErrorCode from daem0nmcp.api.v7.models import ( Cursor, JsonObject, RelativePath, WireModel, _user_text_fields, + is_host_absolute_path, ) from daem0nmcp.api.v7.production import create_v7_server from daem0nmcp.api.v7.resources import ( @@ -35,14 +37,20 @@ RuleResourceDocument, WarningResourceDocument, ) +from daem0nmcp.api.v7.responses import ResponseFactory from daem0nmcp.api.v7.tools import ( TOOL_DATA_MODELS, TOOL_INPUT_MODELS, + CodeIndexInput, DiagnosticSummary, + DocumentIngestUrlInput, + ExportEvent, HealthData, HttpsUrl, MediumText, MemoryStoreInput, + PortableVectorPoint, + SandboxExecutionData, ) from daem0nmcp.config import Settings from daem0nmcp.discovery_projection import ( @@ -57,6 +65,7 @@ GovernanceEventCommand, GovernanceEventStore, ) +from daem0nmcp.retrieval.runtime import await_projection_job_drains from daem0nmcp.retrieval.specialized_projection import SpecializedProjectionBuilder from daem0nmcp.storage_activation import resolve_active_database from tests.api_v7.process_client import initialize_workspaces @@ -75,7 +84,7 @@ # The UD-3 examples: a regex escape, routes, and a Windows file location. PATH_CONTENT = ("\\\\d+", "/health", "GET /api/v1/users", "C:\\proj\\src\\app.py") # Mechanical-coverage seeds (R-26). -SEEDS = ("C:\\proj\\x.py", "/health", "\\d+") +SEEDS = ("C:\\proj\\x.py", "/health", "\\d+", "\x1b[0m") NEUTRAL = "plain neutral words" # A path in text is the user's business; these outcomes mean it was refused. # Readers turn a row that fails its output model into CAPABILITY_DEGRADED. @@ -90,6 +99,7 @@ ) RESOURCE_KINDS = ("warnings", "failures", "rules", "active-context") TOKEN = "preflight-token-placeholder-0001" +WS = "ws_" + "0" * 24 def _invoker(client: Client): @@ -111,7 +121,7 @@ async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: def _store_input(**overrides: Any) -> dict[str, Any]: return { - "workspace_id": "ws_" + "0" * 24, + "workspace_id": WS, "record_type": "decision", "content": "text", "idempotency_key": "path-policy-0001", @@ -165,6 +175,101 @@ def test_server_generated_fields_reject_host_paths(): ) +def test_relative_paths_are_checked_by_their_own_validator(): + """Valid relative names and globs are not mistaken for absolute paths.""" + names = ["src/**/foo.py", "src/__pycache__/x", "docs-/a", "pkg_/a.py"] + index = CodeIndexInput(workspace_id=WS, patterns=names) + assert index.patterns == names + for invalid in ("C:\\x", "/etc/x", "../x", "src/../x", "~/x"): + with pytest.raises(ValidationError): + CodeIndexInput(workspace_id=WS, patterns=[invalid]) + + +def test_ingest_topic_accepts_paths(): + for text in SEEDS: + DocumentIngestUrlInput( + workspace_id=WS, + url="https://example.com/doc", + topic=text, + idempotency_key="path-policy-0001", + preflight_token=TOKEN, + ) + + +def test_error_remedies_echo_path_bearing_arguments(): + """A token error on path-bearing input returns its structured remedy (R-1).""" + for text in SEEDS: + failure = ( + ResponseFactory() + .begin(WS) + .failure( + ErrorCode.COUNSEL_REQUIRED, + "A preflight is required.", + remedy_tool="memory_preflight", + remedy_arguments={ + "workspace_id": WS, + "target_tool": "rule_create", + "target_arguments": {"trigger": text, "must_do": [text]}, + }, + ) + ) + assert failure.error.remedy.arguments["target_arguments"]["trigger"] == text + + +def test_sandbox_output_lines_may_mention_paths(): + line = 'File "/tmp/run.py", line 1, in ' + data = SandboxExecutionData( + success=False, + stdout="", + stderr=line, + exit_status=1, + execution_time_ms=5, + sanitized_logs=[f"stderr: {line}", "x" * 4096], + ) + assert data.sanitized_logs[0].endswith(line) + + +def test_export_structured_fields_stay_guarded(): + """Only user text in export payloads is exempt (R-3, R-4).""" + from daem0nmcp.api.v7.operations import CoreOperationError, _reject_raw_paths + + user = {"record": {"content": "/health", "context": {"note": "C:\\x"}}} + _reject_raw_paths([user]) + for relative in ("../../Users/x/y.py", "d:/other/x.py", "/etc/x"): + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path_relative": relative}}]) + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path": "C:\\proj\\x.py"}}]) + event = { + "event_id": "evt_" + "2" * 64, + "event_type": "memory.created", + "happened_at": "2026-08-08T12:00:00Z", + "content_hash": "a" * 64, + "payload": {"data": {"content": "C:\\proj\\x.py"}, "actor_id": "agent"}, + } + ExportEvent.model_validate(event) + with pytest.raises(ValidationError): + ExportEvent.model_validate( + {**event, "payload": {**event["payload"], "actor_id": "C:\\proj"}} + ) + with pytest.raises(ValidationError): + PortableVectorPoint(point_id="p", payload={"model_id": "C:\\m"}, vector=[1.0]) + + +def test_every_input_field_shape_is_classified(): + """Discovery walks every input model, bundles included, and knows each shape.""" + paths = { + f"{tool}.{'.'.join(map(str, path))}" + for tool, model in TOOL_INPUT_MODELS.items() + for path, _kind_ in _free_text_paths(model) + } + assert { + "workspace_import.bundle.events.0.payload", + "workspace_import.bundle.legacy_rows", + "memory_store_batch.records.0.context", + } <= paths, sorted(paths) + + def _models(root: type[BaseModel], seen: set[type[BaseModel]]) -> None: if root in seen: return @@ -202,9 +307,10 @@ def _sample(annotation: object, text: str) -> object: def test_user_text_round_trips_into_every_output_field_that_carries_it(): """Each string a UserText input accepts validates in every UserText output. - Output fields are found from the models, so a new UserText field is - covered without editing this test. Each probe inherits the model-wide - path check from ``WireModel``. + This shows the exemption works for every output field already typed + ``UserText`` (found from the models, each probe inheriting ``WireModel``'s + path check). It cannot show that the right fields carry the type; the + end-to-end test below does that. """ outputs: set[type[BaseModel]] = set() for model in ( @@ -235,7 +341,7 @@ def test_user_text_round_trips_into_every_output_field_that_carries_it(): probe.model_validate({"value": _sample(annotation, text)}) # The probe is meaningful: a guarded text field refuses the same text. guarded = create_model("Guarded", __base__=WireModel, value=(MediumText, ...)) - if text != "\\\\d+" and text != "\\d+": + if is_host_absolute_path(text): with pytest.raises(ValidationError): guarded.model_validate({"value": text}) @@ -359,6 +465,7 @@ def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[ records=(EntityRecordSeed(record_ids[2 * index]),), ) for index, text in enumerate(texts) + if is_host_absolute_path(text) ), communities=( CommunityProjectionSeed( @@ -452,6 +559,22 @@ async def ok(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: "idempotency_key": f"path-outcome-{index}-{record_type}", }, ) + stale = await invoke( + "memory_store", + { + "workspace_id": ws, + "record_type": "decision", + "content": "GET /health returns 200", + "idempotency_key": "path-stale-token-0001", + "preflight_token": TOKEN, + }, + ) + assert stale["error"]["code"].startswith("TOKEN_"), stale + remedy = stale["error"]["remedy"] + assert remedy["tool"] == "memory_preflight" + assert remedy["arguments"]["target_arguments"]["content"] == ( + "GET /health returns 200" + ) await ok("session_brief", {}) for kind in RESOURCE_KINDS: await client.read_resource(f"memory://workspaces/{ws}/{kind}") @@ -460,15 +583,34 @@ async def ok(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: # --------------------------------------------------------------------------- # Mechanical coverage (R-26). +# An import bundle is hash-chained: editing any one string makes it +# IMPORT_INVALID. Its fields are covered by exporting the seeded workspace +# and importing it back (see the end of the test), not by field seeding. +ROUND_TRIPPED = frozenset({"workspace_import"}) +# The ingest URL is fetched before the topic is read, and tests have no +# network, so every call stops at the URL. The topic's input type is +# covered by test_ingest_topic_accepts_paths. +FETCH_FIRST = frozenset({"document_ingest_url.topic"}) + + +def _mentions_str(annotation: object) -> bool: + return annotation is str or any(_mentions_str(a) for a in get_args(annotation)) + def _kind(annotation: object) -> object: - """``"str"``, ``"json"``, a model class, ``("list", kind)`` or ``None``.""" + """``"str"``, ``"json"``, a model class, ``("list", kind)`` or ``None``. + + Fails on any string-bearing shape it cannot classify, so a new free-text + shape cannot silently drop out of coverage. + """ if annotation in (Cursor, RelativePath, HttpsUrl): return None origin = get_origin(annotation) if origin in (Union, types.UnionType): kinds = {_kind(item) for item in get_args(annotation) if item is not type(None)} - return kinds.pop() if len(kinds) == 1 else None + kinds.discard(None) + assert len(kinds) <= 1, f"mixed union {annotation!r}" + return kinds.pop() if kinds else None if origin is Annotated: base, *metadata = get_args(annotation) if any(getattr(item, "pattern", None) for item in metadata): @@ -476,6 +618,7 @@ def _kind(annotation: object) -> object: return _kind(base) if origin in (list, set, frozenset): inner = _kind(get_args(annotation)[0]) + assert not isinstance(inner, tuple), f"nested list {annotation!r}" return None if inner is None else ("list", inner) if origin is Literal: return None @@ -485,42 +628,65 @@ def _kind(annotation: object) -> object: return "json" if isinstance(annotation, type) and issubclass(annotation, BaseModel): return annotation + assert not _mentions_str(annotation), f"unclassified {annotation!r}" return None -def _free_text_paths(model: type[BaseModel], base: dict[str, Any], prefix=()): - """Every free-text-typed field of ``model``, read from its annotations. - - Nested models are followed where ``base`` supplies them. - """ +def _free_text_paths(model: type[BaseModel], prefix=()): + """Every free-text field of ``model`` and its nested models, by annotation.""" hints = get_type_hints(model, include_extras=True) for name in model.model_fields: kind = _kind(hints[name]) if kind is None: continue - if kind in ("str", "json") or kind == ("list", "str"): + if isinstance(kind, type): + yield from _free_text_paths(kind, (*prefix, name)) + elif isinstance(kind, tuple) and isinstance(kind[1], type): + yield from _free_text_paths(kind[1], (*prefix, name, 0)) + else: yield (*prefix, name), kind - elif isinstance(kind, type) and isinstance(base.get(name), dict): - yield from _free_text_paths(kind, base[name], (*prefix, name)) - elif ( - isinstance(kind, tuple) - and isinstance(kind[1], type) - and isinstance(base.get(name), list) - and base[name] - ): - yield from _free_text_paths(kind[1], base[name][0], (*prefix, name, 0)) -def _seeded(base: dict[str, Any], path: tuple, kind: object, text: str) -> dict: +def _seed_value(kind: object, text: str) -> object: + if kind == "str": + return text + if kind == "json": + return {"note": text, text: [text]} + if kind == ("list", "str"): + return [text] + assert kind == ("list", "json"), kind + return [{"note": text, text: [text]}] + + +def _seeded(tool: str, base: dict[str, Any], path: tuple, kind: object, text: str): + """``base`` with the field at ``path`` set to ``text``, and its siblings fit.""" arguments = copy.deepcopy(base) - target = arguments - for step in path[:-1]: + target: Any = arguments + for step, following in zip(path[:-1], path[1:], strict=True): + if isinstance(step, int): + while len(target) <= step: + target.append({}) + elif step not in target: + target[step] = [] if isinstance(following, int) else {} target = target[step] - value: object = {"json": {"note": text, text: [text]}, "str": text}.get( - kind, - [text], # type: ignore[arg-type] + name = path[-1] + # Exactly one selector is allowed; the seeded name replaces the id. + target.pop( + {"entity_name": "entity_id", "qualified_name": "code_entity_id"}.get(name, ""), + None, ) - target[path[-1]] = value + if name == "procedure_steps": + target["record_type"] = "procedure" + if (tool, path) == ("memory_preflight", ("target_arguments",)): + # Seed the target tool's own argument, not the envelope. + target["target_tool"] = "memory_store" + target["target_arguments"] = { + "record_type": "decision", + "content": text, + "idempotency_key": "mech-preflight-target-0001", + } + return arguments + target[name] = _seed_value(kind, text) return arguments @@ -542,54 +708,117 @@ def _own_read_arguments(arguments: dict[str, Any]) -> dict[str, Any]: async def _seed_free_text(invoke, workspace_id: str, tool: str, arguments: dict): """Call ``tool`` with each free-text field set to each seed. - A path must not change the outcome into a refusal: each seeded call - either matches the same call with neutral text or is not a refusal. + The neutral-text call must not be refused, so a wrongly guarded field + cannot hide behind bad base arguments. (It may miss: a lookup by a name + that does not exist is NOT_FOUND.) Each seeded call must then get the + neutral call's outcome or at least not a refusal. """ + if tool in ROUND_TRIPPED: + return [] base = _own_read_arguments(arguments) labels = [] - for path, kind in _free_text_paths(TOOL_INPUT_MODELS[tool], base): + for path, kind in _free_text_paths(TOOL_INPUT_MODELS[tool]): label = f"{tool}.{'.'.join(map(str, path))}" + if label in FETCH_FIRST: + continue labels.append(label) - neutral = await _target_call( - invoke, - workspace_id, - tool, - _fresh_keys(_seeded(base, path, kind, NEUTRAL), label + NEUTRAL), - ) - for text in SEEDS: - response = await _target_call( - invoke, - workspace_id, - tool, - _fresh_keys(_seeded(base, path, kind, text), label + text), - ) - code = _code(response) - assert code == _code(neutral) or code not in REFUSALS, ( - label, - text, - response["error"], - _code(neutral), - ) + codes = [] + for text in (NEUTRAL, *SEEDS): + call = _fresh_keys(_seeded(tool, base, path, kind, text), label + text) + response = await _target_call(invoke, workspace_id, tool, call) + if _code(response) == "INTERNAL_ERROR": + # SQLite contention with a background projection drain is still + # reported as INTERNAL_ERROR on this base (PR 3 makes it + # retryable). A refusal caused by the text repeats; retry once. + call = _fresh_keys(call, label + text + "retry") + response = await _target_call(invoke, workspace_id, tool, call) + codes.append(_code(response)) + neutral, *seeded = codes + assert neutral not in REFUSALS, (label, neutral) + for text, code in zip(SEEDS, seeded, strict=True): + assert code == neutral or code not in REFUSALS, (label, text, code, neutral) return labels +def _is_write(tool: str) -> bool: + # memory_record_outcome writes a memory event without a preflight; like + # every write it marks the graph stale until the next rebuild. + return _is_protected(tool) or tool == "memory_record_outcome" + + async def _read_everything(invoke, workspace_id: str, seed) -> dict[str, str]: - """Every read tool with the workspace's own objects, then each resource.""" + """Every read tool with the workspace's own objects.""" codes: dict[str, str] = {} for tool, arguments in sorted(_sweep_arguments(seed).items()): - if _is_protected(tool) or tool in CROSS_WORKSPACE_TOOLS: + if _is_write(tool) or tool in CROSS_WORKSPACE_TOOLS: continue plain = _fresh_keys(_own_read_arguments(arguments), f"{tool}-{len(codes)}") codes[tool] = _code(await invoke(tool, {"workspace_id": workspace_id, **plain})) return codes +async def _export_pages(invoke, workspace_id: str) -> list[dict[str, Any]]: + scope = {"workspace_id": workspace_id, "page_byte_limit": 65536} + page = await invoke("workspace_export", scope) + assert page["ok"], page["error"] + pages = [page["data"]] + while not pages[-1]["complete"]: + previous = pages[-1] + page = await invoke( + "workspace_export", + { + **scope, + "export_session_id": previous["export_session_id"], + "page_index": previous["page_index"] + 1, + "cursor": previous["next_cursor"], + }, + ) + assert page["ok"], page["error"] + pages.append(page["data"]) + return pages + + +async def _import_pages(invoke, workspace_id: str, pages) -> dict[str, Any]: + import_id = None + key = "mech-import-0001" + for page in pages: + arguments = {"bundle": page, "finalize": False, "idempotency_key": key} + if import_id is not None: + arguments["import_session_id"] = import_id + staged = await _target_call(invoke, workspace_id, "workspace_import", arguments) + assert staged["ok"], staged["error"] + import_id = staged["data"]["import_session_id"] + finish = {"import_session_id": import_id, "finalize": True, "idempotency_key": key} + done = await _target_call(invoke, workspace_id, "workspace_import", finish) + assert done["ok"], done["error"] + return done["data"] + + +def _texts(value: Any) -> set[str]: + if isinstance(value, dict): + return {text for item in value.values() for text in _texts(item)} + if isinstance(value, list): + return {text for item in value for text in _texts(item)} + return {value} if isinstance(value, str) else set() + + +# Reads that stay gated whatever is stored; every other read must be ok. +GATED_READS = { + # Needs a native edit request, which only the edit hook makes. + "edit_preflight": "NOT_FOUND", + # The default selector exceeds the foreground node budget; the same + # tools are read below with a small selector over the path records. + "knowledge_graph_get": "TASK_REQUIRED", + "knowledge_graph_render": "TASK_REQUIRED", +} + + async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_path): """Seed every free-text input field and DB row with paths; read everything. An input field typed as free text but not ``UserText`` refuses the seed, and an output field carrying it fails or degrades the read; either fails - this test. + this test. The workspace is then exported and imported back. """ a_root, b_root = tmp_path / "alpha", tmp_path / "beta" alpha, beta = await initialize_workspaces((a_root, b_root)) @@ -604,49 +833,89 @@ async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_p invoke = _invoker(client) seed = await _seed_workspace_a(invoke, a, b) await _mint_tokens(invoke, a, b, seed) - # Outcomes before any path is stored. - baseline = await _read_everything(invoke, a, seed) + # Writes first; reads are seeded below, once every kind of row exists + # (a name lookup before any entity exists stops at the graph gate). writes = { - **_sweep_arguments(seed), - "workspace_link": {"linked_workspace_id": b}, + tool: arguments + for tool, arguments in _sweep_arguments(seed).items() + if _is_write(tool) } + writes["workspace_link"] = {"linked_workspace_id": b} for tool, arguments in sorted(writes.items()): seeded += await _seed_free_text(invoke, a, tool, arguments) # Rows as a migrated v6 database or a pre-fix write left them. - _insert_rows(a_root, a, SEEDS) + record_ids = _insert_rows(a_root, a, SEEDS) async with Client(_server((a_root, b_root))) as client: invoke = _invoker(client) assert (await invoke("session_brief", {"workspace_id": a}))["ok"] await _mint_tokens(invoke, a, b, seed) entities = await invoke("entity_list", {"workspace_id": a, "limit": 100}) - assert set(SEEDS) <= {item["name"] for item in entities["data"]["items"]} + path_seeds = {text for text in SEEDS if is_host_absolute_path(text)} + assert path_seeds <= {item["name"] for item in entities["data"]["items"]} seed.ids["entity"] = entities["data"]["items"][0]["entity_id"] # The inserted graph generation replaces the seed's community IDs. communities = await invoke("community_list", {"workspace_id": a}) seed.ids["community"] = communities["data"]["items"][0]["community_id"] + # The stored text comes back verbatim, not dropped or degraded. + # Before any read that writes (outcomes, rebuilds) makes the graph stale. + shown: set[str] = set() + for tool, arguments in ( + ("rule_list", {"limit": 100}), + ("context_trigger_list", {"limit": 100}), + ("memory_search_text", {"query": "Migrated note", "limit": 100}), + ( + "knowledge_graph_render", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ( + "knowledge_graph_get", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ): + response = await invoke(tool, {"workspace_id": a, **arguments}) + assert response["ok"], (tool, response["error"]) + shown |= _texts(response["data"]) + assert set(SEEDS) <= shown, set(SEEDS) - shown + assert {f"Migrated note: {text}" for text in SEEDS} <= shown after = await _read_everything(invoke, a, seed) + assert {t: c for t, c in after.items() if c != "ok"} == GATED_READS, after for tool, arguments in sorted(_sweep_arguments(seed).items()): if tool in after: # Query-like fields searching for the paths. - await _seed_free_text(invoke, a, tool, arguments) + seeded += await _seed_free_text(invoke, a, tool, arguments) for kind in RESOURCE_KINDS: await client.read_resource(f"memory://workspaces/{a}/{kind}") - # Stored paths never turn a read into a failure. (Entity reads use a real - # entity only after the rows exist.) - changed = { - tool: (baseline[tool], code) - for tool, code in after.items() - if code not in {"ok", baseline[tool]} - } - assert not changed, changed - assert after["entity_evolution_trace"] == "ok", after - assert after["context_trigger_list"] == "ok", after - # Nested and cross-workspace inputs are reached too. + # Let background projection drains settle so the export pages form + # one consistent snapshot. + await await_projection_job_drains() + pages = await _export_pages(invoke, a) + + # The exported workspace imports back into empty storage (R-24). + storage = a_root / ".daem0nmcp" / "storage" + storage.rename(a_root / ".daem0nmcp" / "retained-before-import") + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + restored = await _import_pages(invoke, a, pages) + assert restored["root_hash"] == pages[0]["root_hash"] + found = await invoke( + "memory_search_text", + {"workspace_id": a, "query": "Migrated note", "limit": 100}, + ) + assert found["ok"], found["error"] + assert {f"Migrated note: {text}" for text in SEEDS} <= _texts(found["data"]) + + # Nested and cross-workspace inputs are reached too; the floor catches a + # change that quietly drops fields from coverage. assert { "memory_store.context", + "memory_store.procedure_steps", "memory_store_batch.records.0.content", + "memory_preflight.target_arguments", + "entity_evolution_trace.entity_name", "rule_update.patch.must_do", "workspace_link.label", } <= set(seeded), seeded + assert len(seeded) >= 60, len(seeded) diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index e79a5e7..c72d5a5 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -109,9 +109,14 @@ async def guidance(self, *args: object, **kwargs: object) -> object: class _LeakyPreflightService: + """Guidance whose structured path field holds a host path. + + The rule warning also names a path, which user text may do (UD-3); only + ``relative_file_path`` makes the guidance invalid. + """ + async def guidance(self, *args: object, **kwargs: object) -> object: del args, kwargs - # Rule text may mention paths (UD-3); a path field may not. return { "warnings": [r"Inspect D:\private\workspace\notes.txt"], "records": [ @@ -1011,7 +1016,7 @@ def test_failed_preflight_guidance_is_opaque_and_issues_no_token(self) -> None: self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) def test_path_bearing_guidance_fails_before_capability_issue(self) -> None: - # Catches issuing a usable token for a response that must be redacted. + # Catches issuing a usable token for guidance whose path field is invalid. from daem0nmcp.api.v7.errors import ErrorCode from daem0nmcp.api.v7.pinned import build_pinned_handlers from daem0nmcp.api.v7.tools import MemoryPreflightOutput diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 62d6ece..d799b69 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -118,19 +118,29 @@ def _response_model(tool: str) -> type[BaseModel]: return ApiResponse[TOOL_DATA_MODELS[tool]] # type: ignore[name-defined] +# Marks the one piece of user text allowed to name a host path: the note each +# target stores about its own root. +NOTE_MARKER = "sweepnote4b1d" + + def _host_path_leaks( model: type[BaseModel], document: Any, needles: set[str] ) -> list[str]: - """Strings outside ``UserText`` fields that contain a host path needle. + """Strings that contain a host path needle, other than the stored note. - Validating against the response model also refuses any absolute path in - a server-generated field, whatever its spelling. + Every string is scanned, UserText fields included, so server-assembled + user-text fields (excerpts, rendered context, TODO text, mermaid) stay + watched. A hit is allowed only in a UserText field whose text carries the + note's marker. Validating against the response model also refuses any + absolute path in a server-generated field, whatever its spelling. """ validated = model.model_validate(document, strict=False) + guarded = set(guarded_strings(validated)) return [ value - for value in guarded_strings(validated) + for value in _strings(document) if any(needle in value.lower() for needle in needles) + and (value in guarded or NOTE_MARKER not in value) ] @@ -685,7 +695,7 @@ def test_leak_check_exempts_user_text_by_annotation_only(tmp_path): summary = { "record_id": "mem_" + "1" * 64, "record_type": "warning", - "excerpt": f"Target note: see {tmp_path}", + "excerpt": f"Target note {NOTE_MARKER}: see {tmp_path}", "tags": ["/health"], "current_status": "current", "content_hash": "a" * 64, @@ -693,6 +703,11 @@ def test_leak_check_exempts_user_text_by_annotation_only(tmp_path): "updated_at": "2026-08-08T12:00:00Z", } assert _host_path_leaks(RecordSummary, summary, needles) == [] + # A root in user text the server assembled on its own is still caught. + unmarked = {**summary, "excerpt": f"Rendered from {tmp_path}"} + assert _host_path_leaks(RecordSummary, unmarked, needles) == [ + f"Rendered from {tmp_path}" + ] # The same root in a server-generated field is refused by the model... with pytest.raises(ValidationError): _host_path_leaks( @@ -752,7 +767,7 @@ async def run(self, client, name: str, target: str, root: Path) -> None: # UserText fields; the leak check must pass without a name exemption. note = { "record_type": "warning", - "content": f"Target note: see {root / 'src' / 'neutral.py'}", + "content": f"Target note {NOTE_MARKER}: see {root / 'src' / 'neutral.py'}", "idempotency_key": f"sweep-{name}-host-note-0001", } assert (await _protected(invoke, target, "memory_store", note))["ok"] From 88e66ff7f799d835743fc3cf664b4fc968385b84 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 05:56:24 -0400 Subject: [PATCH 35/72] test: mark the sweep note's file name so extracted entities carry it On Linux the graph extracts the note's POSIX path as a file entity, a user-derived name without the marker, which the stricter leak scan then reported. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_process_workspace_isolation_sweep.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index d799b69..c180ca2 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -765,9 +765,12 @@ async def run(self, client, name: str, target: str, root: Path) -> None: assert (await invoke("session_brief", {"workspace_id": target}))["ok"] # The target's own note names its root. Reads echo it only in # UserText fields; the leak check must pass without a name exemption. + # The marker is in the file name too, so a file entity the graph + # extracts from the note (POSIX paths only) carries it as well. + marked = root / "src" / f"{NOTE_MARKER}.py" note = { "record_type": "warning", - "content": f"Target note {NOTE_MARKER}: see {root / 'src' / 'neutral.py'}", + "content": f"Target note {NOTE_MARKER}: see {marked}", "idempotency_key": f"sweep-{name}-host-note-0001", } assert (await _protected(invoke, target, "memory_store", note))["ok"] From 5e731b1a7bd3215526ffbb1fa196342fd330f734 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 05:58:33 -0400 Subject: [PATCH 36/72] test: disable installed profiles explicitly where a test needs them off CI installs the graph and apps extras, which now turn those profiles on. Tests that exercise the disabled path set DAEM0NMCP__ENABLED=false. The portability restore test retries its storage rename within a bounded deadline, since Windows can briefly hold a handle after the server exits. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_graph_operations.py | 7 ++++++- tests/api_v7/test_process_code_impact.py | 8 ++++++-- tests/api_v7/test_process_dreaming.py | 2 ++ tests/api_v7/test_process_portability.py | 13 ++++++++++++- tests/test_vectors.py | 18 +++++++++--------- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/tests/api_v7/test_graph_operations.py b/tests/api_v7/test_graph_operations.py index 12befa2..a8b7336 100644 --- a/tests/api_v7/test_graph_operations.py +++ b/tests/api_v7/test_graph_operations.py @@ -842,7 +842,12 @@ async def test_actual_mcp_graph_profile_enabled_and_disabled(tmp_path, transport "force": False, "idempotency_key": "process-graph-disabled", } - async with process_client(workspace.root, transport) as session: + # The graph extra is installed in CI, so it is on unless explicitly disabled. + async with process_client( + workspace.root, + transport, + environment_overrides={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) token = await _preflight( session, workspace.workspace_id, "entity_backfill", disabled_arguments diff --git a/tests/api_v7/test_process_code_impact.py b/tests/api_v7/test_process_code_impact.py index 4beb9dd..8e7f852 100644 --- a/tests/api_v7/test_process_code_impact.py +++ b/tests/api_v7/test_process_code_impact.py @@ -111,7 +111,11 @@ async def test_production_code_index_search_and_impact(code_workspace, transport async def test_production_code_index_disabled_profile_has_remedy(code_workspace): scope = {"workspace_id": code_workspace.workspace_id} - async with process_client(code_workspace.root, "stdio") as session: + async with process_client( + code_workspace.root, + "stdio", + environment_overrides={"DAEM0NMCP_APPS_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) disabled = await call( session, @@ -122,4 +126,4 @@ async def test_production_code_index_disabled_profile_has_remedy(code_workspace) assert disabled["error"]["code"] == "CAPABILITY_DISABLED" states = disabled["meta"]["capability_states"] assert states[0]["name"] == "apps" - assert "DAEM0NMCP_APPS_ENABLED=true" in states[0]["remediation"] + assert "unless DAEM0NMCP_APPS_ENABLED=false" in states[0]["remediation"] diff --git a/tests/api_v7/test_process_dreaming.py b/tests/api_v7/test_process_dreaming.py index 8954f36..bc391a6 100644 --- a/tests/api_v7/test_process_dreaming.py +++ b/tests/api_v7/test_process_dreaming.py @@ -82,6 +82,8 @@ async def test_production_dreaming_runs_idempotently_and_reports_missing_graph( "DAEM0NMCP_DREAM_PENDING_MIN_AGE_HOURS": "0", "DAEM0NMCP_DREAM_PENDING_EVIDENCE_THRESHOLD": "1", "DAEM0NMCP_DREAM_REVIEW_COOLDOWN_HOURS": "72", + # CI installs the graph extra, which would otherwise turn graph on. + "DAEM0NMCP_GRAPH_ENABLED": "false", } scope = {"workspace_id": workspace.workspace_id} async with process_client( diff --git a/tests/api_v7/test_process_portability.py b/tests/api_v7/test_process_portability.py index 40e135d..b6e542d 100644 --- a/tests/api_v7/test_process_portability.py +++ b/tests/api_v7/test_process_portability.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import sqlite3 from contextlib import closing from datetime import datetime, timedelta, timezone @@ -78,7 +79,17 @@ async def test_production_export_restores_retained_workspace( retained = tmp_path / ".daem0nmcp" / "retained-before-restore" assert storage.resolve().is_relative_to(tmp_path.resolve()) assert retained.resolve().is_relative_to(tmp_path.resolve()) - storage.rename(retained) + # Windows may briefly keep a handle open inside storage after the server + # exits (seen with the graph profile on), so retry within a bounded deadline. + rename_deadline = asyncio.get_running_loop().time() + 15 + while True: + try: + storage.rename(retained) + break + except PermissionError: + if asyncio.get_running_loop().time() >= rename_deadline: + raise + await asyncio.sleep(0.2) async with process_client(tmp_path, transport) as session: await succeed(session, "session_brief", scope) import_id = None diff --git a/tests/test_vectors.py b/tests/test_vectors.py index ac78d0e..f5420a9 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -62,16 +62,16 @@ def test_hybrid_fallback_to_tfidf(self): class TestEncodeDecode: """Test vector encoding and decoding.""" - def test_encode_reports_models_local_remediation_when_unavailable(self): + def test_encode_reports_models_local_remediation_when_unavailable( + self, monkeypatch + ): """The disabled models-local profile exposes its explicit remediation.""" - if not vectors.is_available(): - from daem0nmcp.capabilities import CapabilityUnavailableError - - with pytest.raises( - CapabilityUnavailableError, match="models-local" - ) as error: - vectors.encode("test text") - assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) + from daem0nmcp.capabilities import CapabilityUnavailableError + + monkeypatch.setenv("DAEM0NMCP_MODELS_LOCAL_ENABLED", "false") + with pytest.raises(CapabilityUnavailableError, match="models-local") as error: + vectors.encode("test text") + assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) def test_decode_empty_returns_none(self): """decode returns None for empty bytes.""" From 46008ecd8ad6171f1822a3c4d6bbac6a7b5a1cd4 Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 06:06:12 -0400 Subject: [PATCH 37/72] test: import the launcher before snapshotting sys.modules patch.dict(sys.modules) dropped the launcher module after the first test, so later tests patched a stale module on Linux. Co-Authored-By: Claude Opus 5 --- tests/test_start_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_start_server.py b/tests/test_start_server.py index 3629942..0426467 100644 --- a/tests/test_start_server.py +++ b/tests/test_start_server.py @@ -11,6 +11,10 @@ import pytest +# Import before patch.dict(sys.modules) snapshots it, so the patched launcher is +# the one start_server imports in every test (not a fresh re-import). +import daem0nmcp.api.v7.launcher # noqa: F401 + REPO_ROOT = Path(__file__).resolve().parents[1] From 556c1b414242eb88d33de2832c446d28b52b15ee Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 06:08:07 -0400 Subject: [PATCH 38/72] fix: garbage-collect superseded projection generations Every write rebuilt the lexical and specialized projections into a new generation and never dropped the old one: 400 memories left 803 FTS5 tables and a 176 MB database, and per-write drains and connection opens slowed with the schema bloat (F-001, F-010, F-026, F-039, F-057). After each activation commits, collect_superseded_generations runs in its own short BEGIN IMMEDIATE with a 250 ms busy timeout. It picks ready/failed manifests of that workspace and projection, keeps the active and the most recently deactivated generation (by activated_at, not generation arithmetic), and drops at most 5 per pass: the FTS table, the generation rows and the manifest. Busy or failed GC is logged and skipped; the next activation retries. Graph generations are left alone because discovery tables reference them. GENERATION_TABLES moves to lexical_config so verification and GC share it. Lock contention during a lexical or specialized rebuild is now DATABASE_IN_USE, which the job runner retries, instead of a terminal *_UNAVAILABLE that dead-lettered the job on its first attempt (F-060); projection_rebuild surfaces it as retryable DATABASE_IN_USE. Tests: 50 spaced writes with forced drains stay at <= 2 generations per projection; a 120-generation backlog shrinks 5 per activation; failed or never-activated generations never evict the last active one; GC blocked by an open reader in DELETE journal mode does not fail the rebuild; a contended rebuild is re-queued; memory_preflight spawns no subprocess. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/operations.py | 2 + daem0nmcp/retrieval/lexical_config.py | 9 + daem0nmcp/retrieval/projections.py | 100 +++++ daem0nmcp/retrieval/specialized_projection.py | 16 +- daem0nmcp/verification_v7.py | 10 +- tests/api_v7/test_production.py | 49 +++ tests/test_retrieval_projection_growth.py | 358 ++++++++++++++++++ 7 files changed, 536 insertions(+), 8 deletions(-) create mode 100644 tests/test_retrieval_projection_growth.py diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..14420ef 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -912,6 +912,8 @@ def _projection_sync( raise CoreOperationError("CAPABILITY_DISABLED") from exc if exc.code in {"LEXICAL_UNAVAILABLE", "FTS5_UNAVAILABLE"}: raise CoreOperationError("LEXICAL_UNAVAILABLE") from exc + if exc.code == "DATABASE_IN_USE": + raise CoreOperationError("DATABASE_IN_USE") from exc raise CoreOperationError("CAPABILITY_DEGRADED") from exc except CoreOperationError: raise diff --git a/daem0nmcp/retrieval/lexical_config.py b/daem0nmcp/retrieval/lexical_config.py index d23c243..e87797f 100644 --- a/daem0nmcp/retrieval/lexical_config.py +++ b/daem0nmcp/retrieval/lexical_config.py @@ -18,6 +18,14 @@ "tokenizer": LEXICAL_TOKENIZER, } +# Per-generation row tables of the rebuildable local projections. Temporal +# keeps no rows of its own; graph generations are owned by discovery tables. +GENERATION_TABLES = { + "lexical": "retrieval_documents", + "procedure": "record_procedures", + "outcome": "record_outcome_view", +} + _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _FTS_PARTITION = re.compile(r"^retrieval_fts_[0-9a-f]{24}_g[1-9][0-9]*$") @@ -48,6 +56,7 @@ def lexical_fts_table_name(workspace_id: str, generation: int) -> str: __all__ = [ + "GENERATION_TABLES", "LEXICAL_BM25_WEIGHTS", "LEXICAL_BUILD_CONFIG", "LEXICAL_TOKENIZER", diff --git a/daem0nmcp/retrieval/projections.py b/daem0nmcp/retrieval/projections.py index 4114d29..387960d 100644 --- a/daem0nmcp/retrieval/projections.py +++ b/daem0nmcp/retrieval/projections.py @@ -4,6 +4,7 @@ import hashlib import json +import logging import re import sqlite3 import time @@ -11,14 +12,21 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import ProjectionJobRunner from .lexical_config import ( + GENERATION_TABLES, LEXICAL_TOKENIZER, lexical_build_config_hash, lexical_fts_table_name, ) +from .specialized_contract import procedure_fts_table_name + +logger = logging.getLogger(__name__) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _LEXICAL_BUILDER_VERSION = "retrieval-lexical-1" +_COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) +_GC_BUSY_TIMEOUT_MS = 250 class ProjectionBuildError(RuntimeError): @@ -284,11 +292,19 @@ def rebuild( self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") if isinstance(exc, ProjectionBuildError): raise + if isinstance( + exc, sqlite3.OperationalError + ) and ProjectionJobRunner._is_lock_contention(exc): + raise ProjectionBuildError( + "DATABASE_IN_USE", "lexical projection build was contended" + ) from exc if isinstance(exc, sqlite3.Error): raise ProjectionBuildError( "LEXICAL_UNAVAILABLE", "FTS5 projection build is unavailable" ) from exc raise + if owns_transaction: + collect_superseded_generations(self.connection, workspace_id, "lexical") return result def apply_active_delta(self, workspace_id: str, record_id: str) -> bool: @@ -748,8 +764,92 @@ def _clock_value(self) -> int: return value +def collect_superseded_generations( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + limit: int = 5, +) -> int: + """Drop up to *limit* superseded generations of one local projection. + + Runs after an activation commits, in its own short transaction. Victims + are chosen by manifest status: ``ready`` or ``failed`` rows only, never + the active generation, a ``building`` or ``rebuild_required`` one, or the + most recently deactivated generation (a reader that looked up the old + manifest just before the switch may still be querying it). Best effort: + on contention or any SQLite error it gives up and the next activation + retries. Returns the number of generations removed. + """ + + if ( + projection not in _COLLECTABLE_PROJECTIONS + or _WORKSPACE_ID.fullmatch(workspace_id) is None + or connection.in_transaction + ): + return 0 + generation_table = GENERATION_TABLES.get(projection) + try: + previous_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0]) + connection.execute(f"PRAGMA busy_timeout={_GC_BUSY_TIMEOUT_MS}") + try: + connection.execute("BEGIN IMMEDIATE") + try: + victims = [ + int(row[0]) + for row in connection.execute( + "SELECT generation FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=? " + "AND status IN ('ready','failed') " + "AND manifest_id IS NOT (" + "SELECT manifest_id FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=? " + "AND status='ready' AND activated_at_us IS NOT NULL " + "ORDER BY activated_at_us DESC, generation DESC LIMIT 1) " + "ORDER BY generation LIMIT ?", + (workspace_id, projection, workspace_id, projection, limit), + ).fetchall() + ] + for generation in victims: + fts_table = ( + lexical_fts_table_name(workspace_id, generation) + if projection == "lexical" + else procedure_fts_table_name(workspace_id, generation) + if projection == "procedure" + else None + ) + if fts_table is not None: + connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') + if generation_table is not None: + connection.execute( + f'DELETE FROM "{generation_table}" ' + "WHERE workspace_id=? AND projection_generation=?", + (workspace_id, generation), + ) + connection.execute( + "DELETE FROM projection_manifests WHERE workspace_id=? " + "AND projection_name=? AND generation=?", + (workspace_id, projection, generation), + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.execute(f"PRAGMA busy_timeout={previous_timeout}") + except sqlite3.Error as exc: + if isinstance( + exc, sqlite3.OperationalError + ) and ProjectionJobRunner._is_lock_contention(exc): + logger.debug("projection generation GC skipped: database busy") + else: + logger.warning("projection generation GC failed", exc_info=True) + return 0 + return len(victims) + + __all__ = [ "LexicalProjectionBuilder", "ProjectionBuildError", "ProjectionBuildResult", + "collect_superseded_generations", ] diff --git a/daem0nmcp/retrieval/specialized_projection.py b/daem0nmcp/retrieval/specialized_projection.py index a5d8043..2c226b6 100644 --- a/daem0nmcp/retrieval/specialized_projection.py +++ b/daem0nmcp/retrieval/specialized_projection.py @@ -11,6 +11,8 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import ProjectionJobRunner +from .projections import collect_superseded_generations from .specialized_contract import ( SPECIALIZED_BUILDER_VERSION, specialized_projection_contract, @@ -357,17 +359,29 @@ def rebuild( self.connection.commit() else: self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") - return result except Exception as exc: self._rollback(owns_transaction, savepoint) if isinstance(exc, SpecializedProjectionBuildError): raise + if isinstance( + exc, sqlite3.OperationalError + ) and ProjectionJobRunner._is_lock_contention(exc): + raise SpecializedProjectionBuildError( + "DATABASE_IN_USE", + "specialized projection build was contended", + ) from exc if isinstance(exc, sqlite3.Error): raise SpecializedProjectionBuildError( "PROJECTION_UNAVAILABLE", "specialized projection build is unavailable", ) from exc raise + if owns_transaction and projection_name != "graph": + # Graph generations are referenced by the discovery tables. + collect_superseded_generations( + self.connection, workspace_id, projection_name + ) + return result def active_is_current(self, workspace_id: str, projection_name: str) -> bool: """Return whether the active specialized generation is exact.""" diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index be1003b..56528f9 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -35,6 +35,7 @@ ) from .migrations import MIGRATIONS from .migrations.v7 import _source_row_hash, inventory_database +from .retrieval.lexical_config import GENERATION_TABLES from .schema_version import CURRENT_SCHEMA_VERSION, REQUIRED_V7_SCHEMA_VERSIONS from .storage_activation import ( ActiveDatabasePointer, @@ -92,11 +93,6 @@ "entities", "communities", } -_GENERATION_TABLES = { - "lexical": "retrieval_documents", - "procedure": "record_procedures", - "outcome": "record_outcome_view", -} class VerificationV7Error(RuntimeError): @@ -679,7 +675,7 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: "source_event_root_hash,details_json,generation,row_count," "cursor_recorded_at_us,cursor_event_id FROM projection_manifests" ).fetchall() - for name, generation_table in _GENERATION_TABLES.items(): + for name, generation_table in GENERATION_TABLES.items(): if connection.execute( f'SELECT 1 FROM "{generation_table}" rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " @@ -1115,7 +1111,7 @@ def _refresh_manifests(candidate: Path, workspace_ids: list[str]) -> None: ) # Remove only orphaned local generations in the candidate. Otherwise a # renamed/deleted manifest can leave an FTS table occupying the next ID. - for name, table in _GENERATION_TABLES.items(): + for name, table in GENERATION_TABLES.items(): connection.execute( f'DELETE FROM "{table}" AS rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index 1c71d37..7bfc48b 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -602,5 +602,54 @@ def build_server(_surface, **options): self.assertEqual([False], observed) +async def test_memory_preflight_spawns_no_subprocess(tmp_path): + """Preflight guidance must not pay for git (F-040); session_brief still does.""" + import subprocess + + from fastmcp import Client + + from daem0nmcp.api.v7.production import create_v7_server + from tests.api_v7.process_client import initialize_workspaces + + [workspace] = await initialize_workspaces((tmp_path,)) + spawned: list[list[str]] = [] + + def record_spawn(argv, *args, **kwargs): + spawned.append(list(argv)) + raise FileNotFoundError(argv[0]) + + server = create_v7_server( + "stdio", + settings=Settings( + project_root=str(tmp_path), + workspace_roots=[str(tmp_path)], + dream_enabled=False, + ), + environ={}, + ) + scope = {"workspace_id": workspace.workspace_id} + async with Client(server) as client: + with patch.object(subprocess, "Popen", side_effect=record_spawn): + brief = await client.call_tool("session_brief", scope, raise_on_error=False) + assert brief.structured_content["ok"], brief.structured_content + assert any(argv[0] == "git" for argv in spawned), spawned + spawned.clear() + preflight = await client.call_tool( + "memory_preflight", + { + **scope, + "target_tool": "memory_store", + "target_arguments": { + "record_type": "decision", + "content": "Preflight must not spawn processes.", + "idempotency_key": "preflight-no-subprocess-0001", + }, + }, + raise_on_error=False, + ) + assert preflight.structured_content["ok"], preflight.structured_content + assert spawned == [] + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py new file mode 100644 index 0000000..58694b0 --- /dev/null +++ b/tests/test_retrieval_projection_growth.py @@ -0,0 +1,358 @@ +"""Superseded projection generations are garbage-collected (KD-5, F-001). + +Every write rebuilds the local projections into a new generation. Without GC +each rebuild left a full-corpus FTS5 table, its document rows and a manifest +behind, so storage grew quadratically with the number of memories. +""" + +from __future__ import annotations + +import re +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +WORKSPACE_ID = "ws_0123456789abcdef01234567" +_NO_OPTIONAL_PROFILES = { + "local": "disabled", + "models-local": "disabled", + "graph": "disabled", +} +_LOCAL = ("lexical", "procedure", "outcome", "temporal") + + +def _record(content: str, *, procedure: bool = False) -> dict[str, object]: + return { + "record_type": "procedure" if procedure else "decision", + "legacy_type": None, + "content": content, + "rationale": None, + "context": {"steps": ["Check state.", "Apply change."]} if procedure else {}, + "tags": ["growth"], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "growth-test", + "source_model": None, + "deleted_at_us": None, + } + + +class ProjectionGenerationGrowthTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + from daem0nmcp.migrations.schema import MIGRATIONS + + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "growth.sqlite3" + self.connection = sqlite3.connect(self.path) + self.connection.execute("PRAGMA foreign_keys=ON") + # Pre-PR-8 databases on POSIX still use the rollback journal, where + # readers block a committing writer; GC must stay safe there. + self.connection.execute("PRAGMA journal_mode=DELETE") + for version, _description, statements in MIGRATIONS: + if 16 <= version <= 18: + for statement in statements: + self.connection.execute(statement) + self.connection.commit() + self._sequence = 0 + + def tearDown(self) -> None: + self.connection.close() + self._directory.cleanup() + + def _append(self, *, procedure: bool = False) -> None: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=f"mem_{self._sequence:064x}", + stream_kind="memory", + event_type="memory.created", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": _record( + f"growth memory number {self._sequence}", procedure=procedure + ) + }, + ) + ) + self.connection.commit() + + def _rebuild(self, projection: str) -> None: + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.retrieval.specialized_projection import ( + SpecializedProjectionBuilder, + ) + + if projection == "lexical": + LexicalProjectionBuilder(self.connection).rebuild(WORKSPACE_ID) + else: + SpecializedProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID, projection, force=True + ) + + def _manifests(self, projection: str) -> dict[int, str]: + return { + int(row[0]): str(row[1]) + for row in self.connection.execute( + "SELECT generation,status FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=?", + (WORKSPACE_ID, projection), + ) + } + + def _fts_generations(self, projection: str) -> set[int]: + prefix = ( + "retrieval_procedure_fts_" + if projection == "procedure" + else ("retrieval_fts_") + ) + pattern = re.compile(re.escape(prefix + WORKSPACE_ID[3:]) + r"_g(\d+)") + return { + int(match[1]) + for (name,) in self.connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + if (match := pattern.fullmatch(str(name))) is not None + } + + def _active(self, projection: str) -> int: + [generation] = [ + generation + for generation, status in self._manifests(projection).items() + if status == "active" + ] + return generation + + async def test_spaced_writes_keep_generations_bounded(self) -> None: + from daem0nmcp.retrieval.runtime import drain_projection_jobs + + self._append() + for projection in _LOCAL: + self._rebuild(projection) + writes = 50 + for index in range(writes): + self._append(procedure=index % 5 == 0) + # Every job is due immediately: this is the interactive case of + # writes spaced further apart than the coalescing grace. + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runs = await drain_projection_jobs( + self.path, + config=SimpleNamespace(), + max_jobs=8, + capability_statuses=_NO_OPTIONAL_PROFILES, + ) + self.assertEqual( + {"succeeded"}, + {run.status for run in runs}, + [run.reason for run in runs], + ) + + for projection in _LOCAL: + manifests = self._manifests(projection) + self.assertLessEqual(len(manifests), 2, (projection, manifests)) + self.assertEqual(1, list(manifests.values()).count("active"), projection) + for projection in ("lexical", "procedure"): + tables = self._fts_generations(projection) + self.assertLessEqual(len(tables), 2, (projection, tables)) + self.assertEqual(tables, set(self._manifests(projection)), projection) + live = self.connection.execute( + "SELECT COUNT(*) FROM memory_records WHERE workspace_id=?", + (WORKSPACE_ID,), + ).fetchone()[0] + self.assertEqual(writes + 1, live) + documents = self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents" + ).fetchone()[0] + self.assertLessEqual(documents, 2 * live) + + def test_stale_backlog_shrinks_five_generations_per_activation(self) -> None: + self._append(procedure=True) + for projection in ("lexical", "procedure"): + with self.subTest(projection=projection): + # Seed 120 superseded generations the way a pre-GC database + # accumulated them. + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(121): + self._rebuild(projection) + self.assertEqual(121, len(self._manifests(projection))) + self.assertEqual(121, len(self._fts_generations(projection))) + + previous = 120 + while previous > 1: + self._rebuild(projection) + stale = sum( + status != "active" + for status in self._manifests(projection).values() + ) + self.assertEqual(max(1, previous + 1 - 5), stale) + previous = stale + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests)) + self.assertEqual(set(manifests), self._fts_generations(projection)) + active = self._active(projection) + self.assertEqual({active - 1, active}, set(manifests)) + + def test_failed_or_unactivated_generation_never_evicts_last_active(self) -> None: + from daem0nmcp.event_store import deterministic_id + + self._append() + self._rebuild("lexical") + self._rebuild("lexical") + previous_active = self._active("lexical") + template = self.connection.execute( + "SELECT source_event_count,source_event_root_hash,row_count," + "builder_version FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, previous_active), + ).fetchone() + # Generations just below the next one that never became active: an + # arithmetic "keep the one below the new generation" rule would keep + # these and evict the generation readers were actually using. + for offset, status in ((1, "failed"), (2, "ready")): + generation = previous_active + offset + self.connection.execute( + "INSERT INTO projection_manifests (manifest_id,workspace_id," + "projection_name,generation,projection_version,status," + "source_event_count,source_event_root_hash,row_count," + "builder_version,details_json,started_at_us) " + "VALUES (?,?,'lexical',?,1,?,?,?,?,?,'{}',1)", + ( + deterministic_id("prj", "growth-test", generation), + WORKSPACE_ID, + generation, + status, + *template, + ), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + active = self._active("lexical") + self.assertEqual(previous_active + 3, active) + self.assertEqual({previous_active: "ready", active: "active"}, manifests) + self.assertEqual({previous_active, active}, self._fts_generations("lexical")) + + def test_gc_blocked_by_an_open_reader_does_not_fail_the_rebuild(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + reader = sqlite3.connect(self.path, check_same_thread=False) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + # A reader that looked up an old generation is mid-query when + # the activation's GC starts. + reader.execute("BEGIN") + reader.execute( + f'SELECT COUNT(*) FROM "{lexical_fts_table_name(WORKSPACE_ID, oldest)}"' + ).fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + self.assertEqual([0], collected) + self.assertIn(oldest, self._manifests("lexical")) + self.assertIn(oldest, self._fts_generations("lexical")) + self.assertFalse(self.connection.in_transaction) + finally: + reader.close() + + # The next activation retries and catches up. + self._rebuild("lexical") + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertLessEqual(len(self._manifests("lexical")), 2) + + def test_lock_contention_is_retried_not_dead_lettered(self) -> None: + from daem0nmcp.retrieval.jobs import ProjectionJobRunner + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._append() + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runner_connection = sqlite3.connect(self.path, timeout=0.1) + blocker = sqlite3.connect(self.path, check_same_thread=False) + lexical = LexicalProjectionBuilder(runner_connection) + + def contended_rebuild(workspace_id: str): + blocker.execute("BEGIN IMMEDIATE") + try: + return lexical.rebuild(workspace_id) + finally: + blocker.rollback() + + try: + run = ProjectionJobRunner( + runner_connection, builders={"lexical": contended_rebuild} + ).run_once() + assert run is not None + self.assertEqual("DATABASE_IN_USE", run.reason) + self.assertEqual("queued", run.status) + finally: + blocker.close() + runner_connection.close() + + +class GenerationGcUnitTests(unittest.TestCase): + def test_gc_inside_a_caller_transaction_is_a_no_op(self) -> None: + from daem0nmcp.retrieval.projections import collect_superseded_generations + + connection = sqlite3.connect(":memory:") + try: + connection.execute("CREATE TABLE t(x)") + connection.execute("INSERT INTO t VALUES (1)") + self.assertTrue(connection.in_transaction) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "lexical") + ) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "graph") + ) + finally: + connection.close() + + +if __name__ == "__main__": + unittest.main() From 10bf9d6746e66ba9b07869528b89831c4029c74e Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 06:08:08 -0400 Subject: [PATCH 39/72] fix: report entity recall on a lagging lexical index as retryable memory_recall_entity returned non-retryable CAPABILITY_DEGRADED when retrieval did not return every record the entity points to. While the lexical projection is stale, contended or mid-swap that is transient, so it is now DATABASE_IN_USE (retryable, 250 ms). A current lexical index that still misses records remains CAPABILITY_DEGRADED. The discovery surface process test polls recall with a bounded deadline, as test_process_client does for memory_recall. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/discovery_operations.py | 7 +++ tests/api_v7/test_discovery_operations.py | 59 +++++++++++++++++++ .../api_v7/test_process_discovery_surface.py | 28 +++++---- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..7c0d4b4 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -2093,6 +2093,13 @@ async def _memory_recall_entity( raise DiscoveryOperationError("CAPABILITY_DEGRADED") indexed[record_id] = item.record if set(indexed) != set(selection.record_ids): + # A lexical index that is stale, contended, or mid-swap has not + # caught up with the entity's records yet; the caller may retry. + if not any( + diagnostic.provider == "lexical" and diagnostic.status == "ready" + for diagnostic in result.provider_diagnostics + ): + raise DiscoveryOperationError("DATABASE_IN_USE") raise DiscoveryOperationError("CAPABILITY_DEGRADED") await _run_blocking( dependencies, diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index a40b2d8..08af88b 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1162,6 +1162,65 @@ async def retrieve(self, workspace, query, linked_workspace_ids): self.assertEqual(1, len(second.items)) self.assertNotEqual(first.items[0].record_id, second.items[0].record_id) + async def test_memory_recall_entity_missing_members_retry_only_while_stale( + self, + ) -> None: + """A lexical index still catching up is transient; a current one is not.""" + from daem0nmcp.api.v7.discovery_operations import DiscoveryOperationError + from daem0nmcp.api.v7.models import ( + ProviderDiagnostic, + RetrievalData, + TokenUsage, + ) + + self._activate_discovery() + + class RecallService: + status = "degraded" + + async def retrieve(self, workspace, query, linked_workspace_ids): + return RetrievalData( + provider_diagnostics=[ + ProviderDiagnostic( + provider="lexical", + status=self.status, + manifest_generation=1, + elapsed_ms=0.0, + reason=( + None + if self.status == "ready" + else "LEXICAL_REBUILD_REQUIRED" + ), + returned_count=0, + ) + ], + abstained=True, + abstention_reason="NO_EVIDENCE", + token_usage=TokenUsage( + budget=query.token_budget, + requested=0, + selected=0, + rendered=0, + dropped=0, + ), + ) + + recall = RecallService() + operation = self._operations(recall_service=recall)["memory_recall_entity"] + request = _request( + "memory_recall_entity", + workspace_id=self.workspace.workspace_id, + entity_name="Authentication", + ) + for status, code in ( + ("degraded", "DATABASE_IN_USE"), + ("ready", "CAPABILITY_DEGRADED"), + ): + recall.status = status + with self.assertRaises(DiscoveryOperationError) as raised: + await operation(workspace=self.workspace, request=request) + self.assertEqual(code, raised.exception.code) + async def test_stats_read_the_active_canonical_graph_snapshot(self) -> None: """Counting retained-v6 graph rows would misreport the v7 projection.""" from daem0nmcp.api.v7.tools import KnowledgeGraphStatsData diff --git a/tests/api_v7/test_process_discovery_surface.py b/tests/api_v7/test_process_discovery_surface.py index 2b3f101..3d7e57b 100644 --- a/tests/api_v7/test_process_discovery_surface.py +++ b/tests/api_v7/test_process_discovery_surface.py @@ -2,13 +2,14 @@ from __future__ import annotations +import asyncio import importlib.util import pytest from daem0nmcp.database import DatabaseManager from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed @pytest.fixture @@ -132,15 +133,22 @@ async def test_production_graph_discovery_surface(initialized_workspace, transpo {**scope, "entity_id": entity["entity_id"]}, ) assert traced["entity"]["entity_id"] == entity["entity_id"] - recalled = await succeed( - session, - "memory_recall_entity", - {**scope, "entity_name": entity["name"]}, - ) - assert {item["record_id"] for item in recalled["items"]} & { - first["record_id"], - second["record_id"], - } + # The lexical projection may still be catching up with the link and + # backfill events; recall reports that as retryable until it has. + recall_deadline = asyncio.get_running_loop().time() + 30 + while True: + recalled = await call( + session, + "memory_recall_entity", + {**scope, "entity_name": entity["name"]}, + ) + if recalled["ok"] and { + item["record_id"] for item in recalled["data"]["items"] + } & {first["record_id"], second["record_id"]}: + break + assert recalled["ok"] or recalled["error"]["retryable"], recalled + assert asyncio.get_running_loop().time() < recall_deadline, recalled + await asyncio.sleep(0.25) graph = await succeed( session, "knowledge_graph_get", From 797685eb1e693f621fb889471e5366e9ba0c325b Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 06:37:18 -0400 Subject: [PATCH 40/72] test: legacy recall keeps context that mentions a path Context is user text (UD-3); the provenance fields stay opaque and path-free. Co-Authored-By: Claude Opus 5 --- tests/test_memory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_memory.py b/tests/test_memory.py index 532269b..9f4ff41 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -306,8 +306,15 @@ async def test_v7_legacy_id_keeps_opaque_canonical_origin(self, memory_manager): assert selected["record_id"].startswith("mem_") assert selected["evidence_refs"][0]["record_id"] == selected["record_id"] assert selected["origin_workspace_id"].startswith("ws_") - # Public absolute paths are never re-exposed through legacy context. - assert selected["context"] is None + # Context is user text and may mention paths (UD-3); it comes back as + # stored, while the provenance fields stay opaque and path-free. + assert selected["context"] == {"reason": r"stored beside C:\private\source.py"} + provenance = [ + selected["record_id"], + selected["origin_workspace_id"], + *map(str, selected["evidence_refs"]), + ] + assert not any("private" in value for value in provenance), provenance @pytest.mark.asyncio async def test_failed_decisions_boosted(self, memory_manager): From d0773dd9fd42002cf8bf60b144f7639a09a29bfb Mon Sep 17 00:00:00 2001 From: Devil Date: Sat, 19 Sep 2026 06:55:18 -0400 Subject: [PATCH 41/72] test: expect collected lexical generations after reactivation Reactivation keeps the active and the previously active lexical generation; the older one is now garbage-collected. The growth tests skip fsync so their hundreds of seeding rebuilds stay fast on CI disks. Co-Authored-By: Claude Opus 5 --- tests/test_migrate_v7.py | 3 ++- tests/test_retrieval_projection_growth.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index cb4969d..5710696 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -1320,8 +1320,9 @@ def test_reactivation_preserves_only_the_previous_active_lexical_generation(self self.assertEqual("reactivate", reactivated.action) active = sqlite3.connect(resolve_active_database(storage).path) try: + # Generation 1 is superseded twice over and is collected. self.assertEqual( - [(1, "ready"), (2, "ready"), (3, "active")], + [(2, "ready"), (3, "active")], active.execute( "SELECT generation,status FROM projection_manifests " "WHERE projection_name='lexical' ORDER BY generation" diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py index 58694b0..96b6da6 100644 --- a/tests/test_retrieval_projection_growth.py +++ b/tests/test_retrieval_projection_growth.py @@ -60,6 +60,9 @@ def setUp(self) -> None: # Pre-PR-8 databases on POSIX still use the rollback journal, where # readers block a committing writer; GC must stay safe there. self.connection.execute("PRAGMA journal_mode=DELETE") + # Durability is irrelevant here; skipping fsync keeps the hundreds of + # seeding rebuilds fast on slow CI disks. + self.connection.execute("PRAGMA synchronous=OFF") for version, _description, statements in MIGRATIONS: if 16 <= version <= 18: for statement in statements: From 2c6ac4d441770e1bcc21332dbec1045d369b023a Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 00:20:54 -0400 Subject: [PATCH 42/72] fix: report a busy database during transfer as retryable, not as a bad bundle Export and import do their session bookkeeping in short write transactions. A concurrent writer (a projection drain or a dreaming job, both more active now that graph auto-enables) can hold the SQLite write lock past the 5 s busy timeout, and the catch-all turned that SQLITE_BUSY into a non-retryable IMPORT_INVALID with no remedy and no logged cause. Map a busy database to DATABASE_IN_USE, log the mapped cause, and log the discarded cause behind an INTERNAL_ERROR envelope. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/operations.py | 57 +++++++++++++++++++-- daem0nmcp/api/v7/responses.py | 12 +++++ tests/api_v7/test_portable_projections.py | 17 +++++++ tests/api_v7/test_process_portability.py | 60 +++++++++++++---------- 4 files changed, 117 insertions(+), 29 deletions(-) diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..479f704 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -11,6 +11,7 @@ import asyncio import hashlib import hmac +import logging import os import re import secrets @@ -138,6 +139,8 @@ # This is intentionally not the event loop's default executor. The semaphore # remains owned by the concurrent future after an asyncio waiter is cancelled, # so cancellation cannot release capacity while SQLite still holds a lock. +_LOGGER = logging.getLogger(__name__) + _CORE_OPERATION_WORKERS = BoundedWorkerPool( max_workers=4, thread_name_prefix="daem0nmcp-v7-core", @@ -218,6 +221,42 @@ def _validated_storage_path( return resolved +# SQLITE_BUSY and SQLITE_LOCKED, with their extended result codes. +_SQLITE_BUSY_CODES = frozenset({5, 6, 261, 262, 513, 517}) + + +def _is_database_busy(error: BaseException) -> bool: + if not isinstance(error, sqlite3.OperationalError): + return False + code = getattr(error, "sqlite_errorcode", None) # Python 3.11+ + if isinstance(code, int): + return code in _SQLITE_BUSY_CODES + return "is locked" in str(error) + + +def _portable_failure_code(error: BaseException, operation: str) -> str: + """Name a transfer failure the caller can act on, and log its cause. + + A concurrent writer (a projection drain or a dreaming job) can hold the + SQLite write lock past the busy timeout while a transfer does its session + bookkeeping. That is a transient, retryable condition, not an invalid + bundle, and reporting it as one leaves the caller with no next step. + """ + + seen: set[int] = set() + cause: BaseException | None = error + while cause is not None and id(cause) not in seen: + seen.add(id(cause)) + if _is_database_busy(cause): + _LOGGER.warning("%s deferred: %s", operation, cause) + return "DATABASE_IN_USE" + cause = cause.__cause__ or cause.__context__ + _LOGGER.warning( + "%s rejected the bundle: %s", operation, type(error).__name__, exc_info=error + ) + return "IMPORT_INVALID" + + def _verify_schema(connection: sqlite3.Connection) -> None: try: row = connection.execute( @@ -1028,9 +1067,14 @@ def _export_sync( except CoreOperationError: raise except PortableTransferError as exc: - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_export") + raise CoreOperationError(code) from exc except Exception as exc: - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_export") + ) from exc def _journal_payload(bundle: ExportBundle, merge: bool) -> dict[str, Any]: @@ -1446,7 +1490,10 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_import") + raise CoreOperationError(code) from exc except CoreOperationError: if connection.in_transaction: connection.rollback() @@ -1474,7 +1521,9 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_import") + ) from exc finally: if lease is not None and not committed: with suppress(Exception): diff --git a/daem0nmcp/api/v7/responses.py b/daem0nmcp/api/v7/responses.py index fa51258..449baaf 100644 --- a/daem0nmcp/api/v7/responses.py +++ b/daem0nmcp/api/v7/responses.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import secrets from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass @@ -20,6 +21,8 @@ WorkspaceId, ) +_LOGGER = logging.getLogger(__name__) + T = TypeVar("T") @@ -120,6 +123,15 @@ def failure( def internal_error(self, error: BaseException | None = None) -> ApiResponse[Any]: """Return the one deliberately opaque internal failure envelope.""" + if error is not None: + # The caller-visible envelope stays opaque; the operator needs the + # cause, which was previously discarded without a trace. + _LOGGER.error( + "internal error in request %s: %s", + self.request_id, + type(error).__name__, + exc_info=error, + ) del error return self.failure( ErrorCode.INTERNAL_ERROR, diff --git a/tests/api_v7/test_portable_projections.py b/tests/api_v7/test_portable_projections.py index 0633ffc..721e38e 100644 --- a/tests/api_v7/test_portable_projections.py +++ b/tests/api_v7/test_portable_projections.py @@ -1995,3 +1995,20 @@ def blocking_prepare(*args, **kwargs): assert artifacts == 0 attempts = directory / "attempts" assert not attempts.exists() or not any(attempts.iterdir()) + + +def test_a_concurrent_writer_is_reported_as_retryable_not_as_a_bad_bundle(): + """A held SQLite write lock must not read as an invalid bundle.""" + from daem0nmcp.api.v7.operations import _portable_failure_code + from daem0nmcp.api.v7.portable_projections import PortableTransferError + + busy = sqlite3.OperationalError("database is locked") + wrapped = PortableTransferError("IMPORT_INVALID") + wrapped.__cause__ = busy + + assert _portable_failure_code(busy, "workspace_import") == "DATABASE_IN_USE" + assert _portable_failure_code(wrapped, "workspace_import") == "DATABASE_IN_USE" + assert ( + _portable_failure_code(ValueError("bad page"), "workspace_import") + == "IMPORT_INVALID" + ) diff --git a/tests/api_v7/test_process_portability.py b/tests/api_v7/test_process_portability.py index b6e542d..ee6198d 100644 --- a/tests/api_v7/test_process_portability.py +++ b/tests/api_v7/test_process_portability.py @@ -13,7 +13,7 @@ from daem0nmcp.database import DatabaseManager from daem0nmcp.event_store import EventCommand, EventStore from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed from tests.api_v7.test_process_surface import _preflight, _store @@ -41,6 +41,34 @@ async def _export_pages(session, scope, legacy): return pages +async def _import_retrying_while_busy(session, scope, arguments): + """Import one page, waiting out a concurrent writer's SQLite write lock. + + A projection drain or dreaming job can hold the write lock past the server's + busy timeout, which the transfer reports as the retryable DATABASE_IN_USE. + A real client retries it; so does this test, within a bounded deadline. + """ + + deadline = asyncio.get_running_loop().time() + 30 + while True: + result = await call( + session, + "workspace_import", + { + **scope, + **arguments, + "preflight_token": await _preflight( + session, scope, "workspace_import", arguments + ), + }, + ) + if result["ok"]: + return result["data"] + assert result["error"]["code"] == "DATABASE_IN_USE", result["error"] + assert asyncio.get_running_loop().time() < deadline, result["error"] + await asyncio.sleep(0.5) + + @pytest.mark.parametrize("transport", ["stdio", "streamable-http"]) @pytest.mark.parametrize("legacy", [False, True]) async def test_production_export_restores_retained_workspace( @@ -79,8 +107,10 @@ async def test_production_export_restores_retained_workspace( retained = tmp_path / ".daem0nmcp" / "retained-before-restore" assert storage.resolve().is_relative_to(tmp_path.resolve()) assert retained.resolve().is_relative_to(tmp_path.resolve()) - # Windows may briefly keep a handle open inside storage after the server - # exits (seen with the graph profile on), so retry within a bounded deadline. + # Windows can briefly refuse the rename after the server exits. The cause is + # unconfirmed (the venv launcher's real interpreter is a grandchild the + # harness does not wait for, or a virus scanner reading the fresh files); + # no product code renames storage. Retry within a bounded deadline. rename_deadline = asyncio.get_running_loop().time() + 15 while True: try: @@ -101,17 +131,7 @@ async def test_production_export_restores_retained_workspace( } if import_id is not None: arguments["import_session_id"] = import_id - staged = await succeed( - session, - "workspace_import", - { - **scope, - **arguments, - "preflight_token": await _preflight( - session, scope, "workspace_import", arguments - ), - }, - ) + staged = await _import_retrying_while_busy(session, scope, arguments) import_id = staged["import_session_id"] assert staged["staged_pages"] == index + 1 assert staged["imported"] == 0 @@ -120,17 +140,7 @@ async def test_production_export_restores_retained_workspace( "finalize": True, "idempotency_key": "portable-process-import-0001", } - restored = await succeed( - session, - "workspace_import", - { - **scope, - **finish, - "preflight_token": await _preflight( - session, scope, "workspace_import", finish - ), - }, - ) + restored = await _import_retrying_while_busy(session, scope, finish) assert restored["status"] == "succeeded" assert restored["imported"] == 2 assert restored["root_hash"] == expected_root From ff9dcf51a2658a6e4a4d50c4a9017be0a6ee89bc Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 00:25:38 -0400 Subject: [PATCH 43/72] fix: narrow the retryable recall race and harden generation GC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1. Entity recall (R-1, R-10, R-15, R-20): the retry branch fired whenever no lexical diagnostic was ready, so a dead-lettered rebuild, a missing FTS5 build, or a record that simply does not match the entity name told the client to retry every 250 ms forever. It now asks the database instead: retryable DATABASE_IN_USE only when the missing records are absent from the active lexical generation *and* a rebuild job is queued or running. Everything else stays terminal CAPABILITY_DEGRADED. Generation GC: - keep the highest activated ready generation, not the latest activated_at_us, which a clock step could reorder (R-2, R-9); - also collect rebuild_required generations below the active one, which verify-v7 --repair-projections used to leave behind with a full FTS table and document copy each time (R-8); - one undroppable generation no longer wedges the pass: each victim gets its own savepoint and a non-busy failure skips to the next (R-11); - select only typeof(generation)='integer' rows and delete manifests by manifest_id (R-18); - catch Exception, not just sqlite3.Error, so nothing escapes after the activation commit and marks a successful build failed (R-12, R-19); - _is_lock_contention is now the public jobs.is_lock_contention (R-4), and the caller-side graph guard is gone (R-6). verify-v7 reports a projection_generations inventory (per-projection superseded manifests plus FTS partitions, excluding shadow tables), so a GC that has stopped making progress is visible without reading logs (R-5). New offline command for databases that grew before the GC existed (R-23): `python -m daem0nmcp.cli compact-projections --workspace-id ws_…` takes the exclusive storage lock, loops collect_superseded_generations until it returns 0 for every collectable projection, then VACUUMs, so PR 12's documented VACUUM reclaims the space instead of rewriting dead generations. --no-vacuum drains only. Release inventory regenerated. Tests: the growth drain now runs include_optional=True over 20 writes, so procedure/outcome/temporal really rebuild through the job runner (R-13) and the stale rebuild_required marker is asserted to clear (R-22); generation-row deletion is asserted for all three generation tables against a test-local map, with an outcome record seeded (R-14); new cases cover WAL readers (R-16), repair-left rebuild_required generations, an undroppable victim, offline compaction, and the CLI command; the recall test pins pending, dead-lettered and indexed cases (R-15); the process-surface poll asserts DATABASE_IN_USE (R-17). Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/discovery_operations.py | 73 +++++- daem0nmcp/cli.py | 33 ++- daem0nmcp/retrieval/jobs.py | 39 ++-- daem0nmcp/retrieval/operations.py | 80 ++++++- daem0nmcp/retrieval/projections.py | 161 +++++++++---- daem0nmcp/retrieval/specialized_projection.py | 11 +- daem0nmcp/verification_v7.py | 36 +++ docs/release/v7/inventory.md | 3 +- docs/release/v7/requirements.json | 14 +- tests/api_v7/test_discovery_operations.py | 56 +++-- .../api_v7/test_process_discovery_surface.py | 4 +- tests/test_cli_retrieval.py | 61 +++++ tests/test_retrieval_projection_growth.py | 217 +++++++++++++++++- tests/test_verify_v7.py | 27 +++ 14 files changed, 704 insertions(+), 111 deletions(-) diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 7c0d4b4..3a5c52c 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -2046,6 +2046,56 @@ def reader(connection: sqlite3.Connection) -> _EntitySelection: raise _translate_error(error) from None +def _lexical_catchup_pending_sync( + dependencies: DiscoveryOperationDependencies, + workspace: Workspace, + record_ids: tuple[str, ...], +) -> bool: + """Return whether the lexical projection still owes these records. + + True only when the records are absent from the active lexical generation + *and* a rebuild job is queued or running, i.e. a retry can actually make + progress. A dead-lettered rebuild, a permanently unavailable index, or a + record that simply does not match the entity name is not retryable. + """ + + if not record_ids: + return False + + def reader(connection: sqlite3.Connection) -> bool: + active = connection.execute( + "SELECT generation FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND status='active'", + (workspace.workspace_id,), + ).fetchone() + indexed = 0 + if active is not None: + placeholders = ",".join("?" for _ in record_ids) + indexed = int( + connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + f"AND projection_generation=? AND record_id IN ({placeholders})", + (workspace.workspace_id, int(active[0]), *record_ids), + ).fetchone()[0] + ) + if indexed >= len(record_ids): + return False + return ( + connection.execute( + "SELECT 1 FROM background_jobs WHERE workspace_id=? " + "AND job_type='retrieval.projection_rebuild' " + "AND status IN ('queued','running') LIMIT 1", + (workspace.workspace_id,), + ).fetchone() + is not None + ) + + try: + return bool(_read_snapshot(dependencies, workspace, reader)) + except Exception: + return False + + def _generation_is_current_sync( dependencies: DiscoveryOperationDependencies, workspace: Workspace, @@ -2093,14 +2143,21 @@ async def _memory_recall_entity( raise DiscoveryOperationError("CAPABILITY_DEGRADED") indexed[record_id] = item.record if set(indexed) != set(selection.record_ids): - # A lexical index that is stale, contended, or mid-swap has not - # caught up with the entity's records yet; the caller may retry. - if not any( - diagnostic.provider == "lexical" and diagnostic.status == "ready" - for diagnostic in result.provider_diagnostics - ): - raise DiscoveryOperationError("DATABASE_IN_USE") - raise DiscoveryOperationError("CAPABILITY_DEGRADED") + missing = tuple( + record_id + for record_id in selection.record_ids + if record_id not in indexed + ) + # Only a projection that has genuinely not caught up yet is worth + # retrying. A permanently unavailable or dead-lettered lexical + # index stays a terminal CAPABILITY_DEGRADED. + pending = await _run_blocking( + dependencies, + lambda: _lexical_catchup_pending_sync(dependencies, workspace, missing), + ) + raise DiscoveryOperationError( + "DATABASE_IN_USE" if pending else "CAPABILITY_DEGRADED" + ) await _run_blocking( dependencies, lambda: _generation_is_current_sync( diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..ed157ed 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -378,6 +378,19 @@ def main(): "--workspace-id", required=True, help="Registered opaque workspace ID" ) + compact_projections_parser = subparsers.add_parser( + "compact-projections", + help="Offline: drop every superseded projection generation and VACUUM", + ) + compact_projections_parser.add_argument( + "--workspace-id", required=True, help="Registered opaque workspace ID" + ) + compact_projections_parser.add_argument( + "--no-vacuum", + action="store_true", + help="Drop the superseded generations without reclaiming the file", + ) + rebuild_projection_parser = subparsers.add_parser( "rebuild-projection", help="Build and activate one v7 retrieval projection" ) @@ -770,11 +783,16 @@ def main(): print(f"Runs: {len(payload['runs'])}") sys.exit(exit_code) - if args.command in {"projection-status", "rebuild-projection"}: + if args.command in { + "compact-projections", + "projection-status", + "rebuild-projection", + }: import sqlite3 from .retrieval.operations import ( ProjectionOperationError, + compact_projections, projection_status, rebuild_projection, ) @@ -802,7 +820,12 @@ def main(): update={"project_root": str(workspace.root)} ) storage_path = Path(workspace_settings.get_storage_path()) - with DatabaseFileLock(storage_path, "shared"): + # Compaction drops generations a running server could be reading, + # so it takes the offline exclusive lock. + lock_mode = ( + "exclusive" if args.command == "compact-projections" else "shared" + ) + with DatabaseFileLock(storage_path, lock_mode): active = resolve_active_database(storage_path) if active.format_version != 7: raise ProjectionOperationError("FORMAT_7_REQUIRED") @@ -813,6 +836,12 @@ def main(): connection.execute("PRAGMA foreign_keys=ON") if args.command == "projection-status": payload = projection_status(connection, workspace.workspace_id) + elif args.command == "compact-projections": + payload = compact_projections( + connection, + workspace.workspace_id, + vacuum=not args.no_vacuum, + ) else: builders = create_projection_builders( connection, diff --git a/daem0nmcp/retrieval/jobs.py b/daem0nmcp/retrieval/jobs.py index 4376bbd..f9f7a60 100644 --- a/daem0nmcp/retrieval/jobs.py +++ b/daem0nmcp/retrieval/jobs.py @@ -26,6 +26,26 @@ _HEARTBEAT_SHUTDOWN_GRACE_SECONDS = 1.5 +def is_lock_contention(exc: BaseException) -> bool: + """Return whether *exc* is SQLite lock contention a caller may retry.""" + + if not isinstance(exc, sqlite3.OperationalError): + return False + code = getattr(exc, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + } + # sqlite_errorcode is unavailable on supported Python 3.10 builds. + # Match only SQLite's fixed lock diagnostics, never arbitrary detail. + return str(exc).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + class ProjectionJobError(RuntimeError): """Owned, sanitized projection-job failure.""" @@ -250,7 +270,7 @@ def _heartbeat_loop( break except sqlite3.OperationalError as exc: connection.rollback() - if not self._is_lock_contention(exc): + if not is_lock_contention(exc): raise if stop.is_set(): if stop_seen_at is None: @@ -554,22 +574,6 @@ def _now(self) -> int: raise ProjectionJobError("PROJECTION_JOB_CLOCK_INVALID") return value - @staticmethod - def _is_lock_contention(exc: sqlite3.OperationalError) -> bool: - code = getattr(exc, "sqlite_errorcode", None) - if isinstance(code, int): - return code & 0xFF in { - sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_LOCKED, - } - # sqlite_errorcode is unavailable on supported Python 3.10 builds. - # Match only SQLite's fixed lock diagnostics, never arbitrary detail. - return str(exc).casefold() in { - "database is locked", - "database table is locked", - "database schema is locked", - } - def create_projection_job_runner( connection: sqlite3.Connection, @@ -593,4 +597,5 @@ def create_projection_job_runner( "ProjectionJobRun", "ProjectionJobRunner", "create_projection_job_runner", + "is_lock_contention", ] diff --git a/daem0nmcp/retrieval/operations.py b/daem0nmcp/retrieval/operations.py index ab48297..08935e2 100644 --- a/daem0nmcp/retrieval/operations.py +++ b/daem0nmcp/retrieval/operations.py @@ -9,7 +9,12 @@ from dataclasses import asdict from typing import Any -from .projections import LexicalProjectionBuilder, ProjectionBuildError +from .projections import ( + COLLECTABLE_PROJECTIONS, + LexicalProjectionBuilder, + ProjectionBuildError, + collect_superseded_generations, +) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _PROJECTIONS = frozenset( @@ -157,8 +162,81 @@ def projection_status( } +def compact_projections( + connection: sqlite3.Connection, + workspace_id: str, + *, + vacuum: bool = True, +) -> dict[str, Any]: + """Drain every superseded generation offline, then reclaim the file. + + Online activations collect at most five generations each, so a database + that grew before the garbage collector existed needs about a hundred + writes to drain — and a ``VACUUM`` before that only rewrites the dead + generations. This runs the same collector in a loop until it reports + nothing left, so the following ``VACUUM`` actually reclaims the space. + The caller owns the offline exclusive lock; no activation may run + concurrently. + """ + + if not isinstance(connection, sqlite3.Connection): + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + _validate_workspace(workspace_id) + if connection.in_transaction: + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + collected: dict[str, int] = {} + try: + before = _generation_counts(connection, workspace_id) + for projection in sorted(COLLECTABLE_PROJECTIONS): + removed = 0 + while True: + passed = collect_superseded_generations( + connection, workspace_id, projection + ) + if passed == 0: + break + removed += passed + collected[projection] = removed + if vacuum: + connection.execute("VACUUM") + after = _generation_counts(connection, workspace_id) + except sqlite3.Error as exc: + raise ProjectionOperationError("PROJECTION_COMPACTION_FAILED") from exc + return { + "collected": collected, + "superseded_manifests_after": after[0], + "superseded_manifests_before": before[0], + "vacuumed": bool(vacuum), + "workspace_id": workspace_id, + } + + +def _generation_counts( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[int, int]: + placeholders = ",".join("?" for _ in COLLECTABLE_PROJECTIONS) + projections = sorted(COLLECTABLE_PROJECTIONS) + superseded = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders}) AND status<>'active'", + (workspace_id, *projections), + ).fetchone()[0] + ) + total = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders})", + (workspace_id, *projections), + ).fetchone()[0] + ) + return superseded, total + + __all__ = [ "ProjectionOperationError", + "compact_projections", "projection_status", "rebuild_projection", ] diff --git a/daem0nmcp/retrieval/projections.py b/daem0nmcp/retrieval/projections.py index 387960d..2548719 100644 --- a/daem0nmcp/retrieval/projections.py +++ b/daem0nmcp/retrieval/projections.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json -from .jobs import ProjectionJobRunner +from .jobs import is_lock_contention from .lexical_config import ( GENERATION_TABLES, LEXICAL_TOKENIZER, @@ -25,7 +25,8 @@ _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _LEXICAL_BUILDER_VERSION = "retrieval-lexical-1" -_COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) +# Graph is excluded: the discovery tables reference its generations. +COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) _GC_BUSY_TIMEOUT_MS = 250 @@ -292,9 +293,7 @@ def rebuild( self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") if isinstance(exc, ProjectionBuildError): raise - if isinstance( - exc, sqlite3.OperationalError - ) and ProjectionJobRunner._is_lock_contention(exc): + if is_lock_contention(exc): raise ProjectionBuildError( "DATABASE_IN_USE", "lexical projection build was contended" ) from exc @@ -764,6 +763,61 @@ def _clock_value(self) -> int: return value +_VICTIM_QUERY = """ +SELECT manifest_id, generation FROM projection_manifests +WHERE workspace_id=? AND projection_name=? AND typeof(generation)='integer' + AND ( + ( + status IN ('ready','failed') + AND manifest_id IS NOT ( + SELECT manifest_id FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='ready' + AND activated_at_us IS NOT NULL + ORDER BY generation DESC LIMIT 1 + ) + ) + OR ( + status='rebuild_required' + AND generation < COALESCE(( + SELECT generation FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='active' + AND typeof(generation)='integer' + ), -1) + ) + ) +ORDER BY generation LIMIT ? +""" + + +def _drop_generation( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + manifest_id: str, + generation: int, +) -> None: + fts_table = ( + lexical_fts_table_name(workspace_id, generation) + if projection == "lexical" + else procedure_fts_table_name(workspace_id, generation) + if projection == "procedure" + else None + ) + if fts_table is not None: + connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') + generation_table = GENERATION_TABLES.get(projection) + if generation_table is not None: + connection.execute( + f'DELETE FROM "{generation_table}" ' + "WHERE workspace_id=? AND projection_generation=?", + (workspace_id, generation), + ) + connection.execute( + "DELETE FROM projection_manifests WHERE manifest_id=?", + (manifest_id,), + ) + + def collect_superseded_generations( connection: sqlite3.Connection, workspace_id: str, @@ -773,81 +827,86 @@ def collect_superseded_generations( """Drop up to *limit* superseded generations of one local projection. Runs after an activation commits, in its own short transaction. Victims - are chosen by manifest status: ``ready`` or ``failed`` rows only, never - the active generation, a ``building`` or ``rebuild_required`` one, or the - most recently deactivated generation (a reader that looked up the old - manifest just before the switch may still be querying it). Best effort: - on contention or any SQLite error it gives up and the next activation + are chosen by manifest status: ``ready`` and ``failed`` rows, plus + ``rebuild_required`` rows below the active generation (readers consult + those only while no active manifest exists). It never touches the active + or a ``building`` generation, nor the most recently deactivated one (the + highest activated ``ready`` generation), because a reader that resolved + the old manifest just before the switch may still be querying it. + + Best effort: one damaged generation is skipped rather than wedging the + rest, contention ends the pass, and nothing raises — the next activation retries. Returns the number of generations removed. """ if ( - projection not in _COLLECTABLE_PROJECTIONS + projection not in COLLECTABLE_PROJECTIONS or _WORKSPACE_ID.fullmatch(workspace_id) is None or connection.in_transaction ): return 0 - generation_table = GENERATION_TABLES.get(projection) + collected = 0 try: previous_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0]) connection.execute(f"PRAGMA busy_timeout={_GC_BUSY_TIMEOUT_MS}") try: connection.execute("BEGIN IMMEDIATE") try: - victims = [ - int(row[0]) - for row in connection.execute( - "SELECT generation FROM projection_manifests " - "WHERE workspace_id=? AND projection_name=? " - "AND status IN ('ready','failed') " - "AND manifest_id IS NOT (" - "SELECT manifest_id FROM projection_manifests " - "WHERE workspace_id=? AND projection_name=? " - "AND status='ready' AND activated_at_us IS NOT NULL " - "ORDER BY activated_at_us DESC, generation DESC LIMIT 1) " - "ORDER BY generation LIMIT ?", - (workspace_id, projection, workspace_id, projection, limit), - ).fetchall() - ] - for generation in victims: - fts_table = ( - lexical_fts_table_name(workspace_id, generation) - if projection == "lexical" - else procedure_fts_table_name(workspace_id, generation) - if projection == "procedure" - else None - ) - if fts_table is not None: - connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') - if generation_table is not None: + victims = connection.execute( + _VICTIM_QUERY, + ( + workspace_id, + projection, + workspace_id, + projection, + workspace_id, + projection, + limit, + ), + ).fetchall() + for manifest_id, generation in victims: + connection.execute("SAVEPOINT retrieval_generation_gc") + try: + _drop_generation( + connection, + workspace_id, + projection, + str(manifest_id), + int(generation), + ) + except Exception as exc: connection.execute( - f'DELETE FROM "{generation_table}" ' - "WHERE workspace_id=? AND projection_generation=?", - (workspace_id, generation), + "ROLLBACK TO SAVEPOINT retrieval_generation_gc" ) - connection.execute( - "DELETE FROM projection_manifests WHERE workspace_id=? " - "AND projection_name=? AND generation=?", - (workspace_id, projection, generation), - ) + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + if is_lock_contention(exc): + raise + # A damaged generation must not wedge every later + # one; skip past it and keep collecting. + logger.warning( + "projection generation could not be collected", + exc_info=True, + ) + continue + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + collected += 1 connection.commit() except BaseException: connection.rollback() raise finally: connection.execute(f"PRAGMA busy_timeout={previous_timeout}") - except sqlite3.Error as exc: - if isinstance( - exc, sqlite3.OperationalError - ) and ProjectionJobRunner._is_lock_contention(exc): + except Exception as exc: + if is_lock_contention(exc): logger.debug("projection generation GC skipped: database busy") else: logger.warning("projection generation GC failed", exc_info=True) return 0 - return len(victims) + return collected __all__ = [ + "COLLECTABLE_PROJECTIONS", "LexicalProjectionBuilder", "ProjectionBuildError", "ProjectionBuildResult", diff --git a/daem0nmcp/retrieval/specialized_projection.py b/daem0nmcp/retrieval/specialized_projection.py index 2c226b6..1282ee5 100644 --- a/daem0nmcp/retrieval/specialized_projection.py +++ b/daem0nmcp/retrieval/specialized_projection.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json -from .jobs import ProjectionJobRunner +from .jobs import is_lock_contention from .projections import collect_superseded_generations from .specialized_contract import ( SPECIALIZED_BUILDER_VERSION, @@ -363,9 +363,7 @@ def rebuild( self._rollback(owns_transaction, savepoint) if isinstance(exc, SpecializedProjectionBuildError): raise - if isinstance( - exc, sqlite3.OperationalError - ) and ProjectionJobRunner._is_lock_contention(exc): + if is_lock_contention(exc): raise SpecializedProjectionBuildError( "DATABASE_IN_USE", "specialized projection build was contended", @@ -376,8 +374,9 @@ def rebuild( "specialized projection build is unavailable", ) from exc raise - if owns_transaction and projection_name != "graph": - # Graph generations are referenced by the discovery tables. + if owns_transaction: + # Graph is not collectable: the discovery tables reference its + # generations. collect_superseded_generations enforces that. collect_superseded_generations( self.connection, workspace_id, projection_name ) diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index 56528f9..eed1d3d 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -660,6 +660,34 @@ def _verify_mappings(connection: sqlite3.Connection, replay: sqlite3.Connection) return count +def _generation_inventory(connection: sqlite3.Connection) -> dict[str, int]: + """Count superseded local generations so a stalled GC is visible.""" + + from .retrieval.projections import COLLECTABLE_PROJECTIONS + + projections = sorted(COLLECTABLE_PROJECTIONS) + placeholders = ",".join("?" for _ in projections) + superseded = { + str(row[0]): int(row[1]) + for row in connection.execute( + "SELECT projection_name,COUNT(*) FROM projection_manifests " + f"WHERE projection_name IN ({placeholders}) AND status<>'active' " + "GROUP BY projection_name", + projections, + ) + } + inventory = {name: superseded.get(name, 0) for name in projections} + inventory["fts_partitions"] = int( + connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*' " + "AND (name GLOB 'retrieval_fts_*_g*' " + "OR name GLOB 'retrieval_procedure_fts_*_g*')" + ).fetchone()[0] + ) + return inventory + + def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: from .retrieval.projections import LexicalProjectionBuilder from .retrieval.specialized_projection import SpecializedProjectionBuilder @@ -930,6 +958,14 @@ def _verify_database( ) except Exception as exc: checks["projection_manifests"] = _check(False, error=type(exc).__name__) + try: + # Superseded generations should stay near zero. A number that + # keeps growing means the activation-time GC is failing. + checks["projection_generations"] = _check( + True, **_generation_inventory(source) + ) + except Exception as exc: + checks["projection_generations"] = _check(False, error=type(exc).__name__) try: invalid = int( source.execute( diff --git a/docs/release/v7/inventory.md b/docs/release/v7/inventory.md index db82039..cc683e0 100644 --- a/docs/release/v7/inventory.md +++ b/docs/release/v7/inventory.md @@ -2,7 +2,7 @@ Generated by `python scripts/v7_release_inventory.py --write`. Registration is not acceptance; only an `accepted` requirement with explicit evidence metadata can be accepted. -Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **571**. +Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **572**. ## Release ledger @@ -131,6 +131,7 @@ Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resou | hook | `hooks/daem0n_stop_hook.py` | `hooks/daem0n_stop_hook.py` | Hook hooks/daem0n_stop_hook.py runs with its documented input and redacted failure behavior. — pending evidence | pending | | cli | `briefing` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli briefing parses and exercises its documented result contract. — pending evidence | pending | | cli | `check` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli check parses and exercises its documented result contract. — pending evidence | pending | +| cli | `compact-projections` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract. — pending evidence | pending | | cli | `index` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli index parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-claude-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-claude-hooks parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-hooks parses and exercises its documented result contract. — pending evidence | pending | diff --git a/docs/release/v7/requirements.json b/docs/release/v7/requirements.json index 6346d97..8b452d4 100644 --- a/docs/release/v7/requirements.json +++ b/docs/release/v7/requirements.json @@ -4370,6 +4370,16 @@ "package": "daem0nmcp.cli", "status": "pending" }, + { + "acceptance": "python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract.", + "entry_point": "daem0nmcp.cli:main", + "evidence": [], + "id": "cli.compact-projections", + "kind": "cli", + "owner": "cli", + "package": "daem0nmcp.cli", + "status": "pending" + }, { "acceptance": "python -m daem0nmcp.cli index parses and exercises its documented result contract.", "entry_point": "daem0nmcp.cli:main", @@ -5744,8 +5754,8 @@ "tests/test_legacy_vector_migration_guards.py" ], "observed_surface": { - "atomic_requirements": 571, - "cli_commands": 20, + "atomic_requirements": 572, + "cli_commands": 21, "dependency_profiles": 10, "hooks": 15, "registered_tools": 75, diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index 08af88b..eab8404 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1162,35 +1162,32 @@ async def retrieve(self, workspace, query, linked_workspace_ids): self.assertEqual(1, len(second.items)) self.assertNotEqual(first.items[0].record_id, second.items[0].record_id) - async def test_memory_recall_entity_missing_members_retry_only_while_stale( + async def test_memory_recall_entity_retries_only_a_pending_catch_up( self, ) -> None: - """A lexical index still catching up is transient; a current one is not.""" + """Only a projection that still owes the records is worth retrying.""" from daem0nmcp.api.v7.discovery_operations import DiscoveryOperationError from daem0nmcp.api.v7.models import ( ProviderDiagnostic, RetrievalData, TokenUsage, ) + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder self._activate_discovery() class RecallService: - status = "degraded" + """Returns nothing, as a lexical miss on the entity name does.""" async def retrieve(self, workspace, query, linked_workspace_ids): return RetrievalData( provider_diagnostics=[ ProviderDiagnostic( provider="lexical", - status=self.status, - manifest_generation=1, + status="unavailable", + manifest_generation=None, elapsed_ms=0.0, - reason=( - None - if self.status == "ready" - else "LEXICAL_REBUILD_REQUIRED" - ), + reason="LEXICAL_UNAVAILABLE", returned_count=0, ) ], @@ -1205,21 +1202,44 @@ async def retrieve(self, workspace, query, linked_workspace_ids): ), ) - recall = RecallService() - operation = self._operations(recall_service=recall)["memory_recall_entity"] + operation = self._operations(recall_service=RecallService())[ + "memory_recall_entity" + ] request = _request( "memory_recall_entity", workspace_id=self.workspace.workspace_id, entity_name="Authentication", ) - for status, code in ( - ("degraded", "DATABASE_IN_USE"), - ("ready", "CAPABILITY_DEGRADED"), - ): - recall.status = status + + async def code() -> str: with self.assertRaises(DiscoveryOperationError) as raised: await operation(workspace=self.workspace, request=request) - self.assertEqual(code, raised.exception.code) + return raised.exception.code + + # The records are not in the lexical projection yet and its rebuild + # job is queued: a retry can make progress. + self.assertEqual("DATABASE_IN_USE", await code()) + + # The rebuild dead-lettered, so nothing will catch up on its own. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute( + "UPDATE background_jobs SET status='dead_letter'," + "finished_at_us=1 WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + + # The projection holds every member, so the miss is genuine even + # though lexical reports itself unavailable and a job is queued. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + LexicalProjectionBuilder(connection).rebuild(self.workspace.workspace_id) + connection.execute( + "UPDATE background_jobs SET status='queued',finished_at_us=NULL " + "WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) async def test_stats_read_the_active_canonical_graph_snapshot(self) -> None: """Counting retained-v6 graph rows would misreport the v7 projection.""" diff --git a/tests/api_v7/test_process_discovery_surface.py b/tests/api_v7/test_process_discovery_surface.py index 3d7e57b..60e9377 100644 --- a/tests/api_v7/test_process_discovery_surface.py +++ b/tests/api_v7/test_process_discovery_surface.py @@ -146,7 +146,9 @@ async def test_production_graph_discovery_surface(initialized_workspace, transpo item["record_id"] for item in recalled["data"]["items"] } & {first["record_id"], second["record_id"]}: break - assert recalled["ok"] or recalled["error"]["retryable"], recalled + if not recalled["ok"]: + assert recalled["error"]["code"] == "DATABASE_IN_USE", recalled + assert recalled["error"]["retryable"], recalled assert asyncio.get_running_loop().time() < recall_deadline, recalled await asyncio.sleep(0.25) graph = await succeed( diff --git a/tests/test_cli_retrieval.py b/tests/test_cli_retrieval.py index 057f712..5c60a42 100644 --- a/tests/test_cli_retrieval.py +++ b/tests/test_cli_retrieval.py @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from tests.test_migrate_v7 import _create_legacy_database @@ -157,6 +158,66 @@ def test_registered_secondary_workspace_opens_its_own_active_database(self): self.assertEqual(1, len(lexical)) self.assertEqual(second_id, payload["workspace_id"]) + def test_compact_projections_drains_superseded_generations_offline(self): + """Databases bloated before the GC existed need a one-shot drain.""" + + import sqlite3 + + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.storage_activation import resolve_active_database + + with tempfile.TemporaryDirectory() as raw: + root, workspace_id = self._migrated_fixture(raw) + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(9): + LexicalProjectionBuilder(connection).rebuild(workspace_id) + superseded = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical' AND status<>'active'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(9, superseded) + + result = self._run( + root, + "compact-projections", + "--workspace-id", + workspace_id, + ) + + self.assertEqual(0, result.returncode, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(8, payload["collected"]["lexical"]) + self.assertLess( + payload["superseded_manifests_after"], + payload["superseded_manifests_before"], + ) + self.assertTrue(payload["vacuumed"]) + self.assertNotIn(str(root), result.stdout) + connection = sqlite3.connect(database) + try: + remaining = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical'" + ).fetchone()[0] + tables = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND name GLOB 'retrieval_fts_*_g*' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(2, remaining) + self.assertEqual(2, tables) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py index 96b6da6..64014fb 100644 --- a/tests/test_retrieval_projection_growth.py +++ b/tests/test_retrieval_projection_growth.py @@ -22,6 +22,13 @@ "graph": "disabled", } _LOCAL = ("lexical", "procedure", "outcome", "temporal") +# Spelled out here on purpose: importing the production mapping would make a +# shrunken GENERATION_TABLES pass this file. +_GENERATION_TABLES = { + "lexical": "retrieval_documents", + "outcome": "record_outcome_view", + "procedure": "record_procedures", +} def _record(content: str, *, procedure: bool = False) -> dict[str, object]: @@ -74,14 +81,39 @@ def tearDown(self) -> None: self.connection.close() self._directory.cleanup() - def _append(self, *, procedure: bool = False) -> None: + def _record_outcome(self, record_id: str) -> None: from daem0nmcp.event_store import EventCommand, EventStore self._sequence += 1 EventStore(self.connection).append_and_project( EventCommand( workspace_id=WORKSPACE_ID, - stream_id=f"mem_{self._sequence:064x}", + stream_id=record_id, + stream_kind="memory", + event_type="memory.outcome_recorded", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": { + **_record("growth memory number 1", procedure=True), + "outcome": "the generation was collected", + "worked": True, + } + }, + ) + ) + self.connection.commit() + + def _append(self, *, procedure: bool = False) -> str: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + record_id = f"mem_{self._sequence:064x}" + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, stream_kind="memory", event_type="memory.created", occurred_at_us=100 + self._sequence, @@ -95,6 +127,7 @@ def _append(self, *, procedure: bool = False) -> None: ) ) self.connection.commit() + return record_id def _rebuild(self, projection: str) -> None: from daem0nmcp.retrieval.projections import LexicalProjectionBuilder @@ -145,10 +178,11 @@ def _active(self, projection: str) -> int: async def test_spaced_writes_keep_generations_bounded(self) -> None: from daem0nmcp.retrieval.runtime import drain_projection_jobs - self._append() + first = self._append(procedure=True) + self._record_outcome(first) for projection in _LOCAL: self._rebuild(projection) - writes = 50 + writes = 20 for index in range(writes): self._append(procedure=index % 5 == 0) # Every job is due immediately: this is the interactive case of @@ -159,6 +193,9 @@ async def test_spaced_writes_keep_generations_bounded(self) -> None: self.path, config=SimpleNamespace(), max_jobs=8, + # The real drain: every local projection rebuilds, so the + # specialized GC call site runs on the production path. + include_optional=True, capability_statuses=_NO_OPTIONAL_PROFILES, ) self.assertEqual( @@ -166,6 +203,10 @@ async def test_spaced_writes_keep_generations_bounded(self) -> None: {run.status for run in runs}, [run.reason for run in runs], ) + self.assertEqual( + set(_LOCAL), + {name for run in runs for name in run.projections}, + ) for projection in _LOCAL: manifests = self._manifests(projection) @@ -184,6 +225,34 @@ async def test_spaced_writes_keep_generations_bounded(self) -> None: "SELECT COUNT(*) FROM retrieval_documents" ).fetchone()[0] self.assertLessEqual(documents, 2 * live) + self._assert_generation_rows_collected() + # The stale marker every write sets clears once the rebuild runs, and + # GC never leaves it stuck on an active manifest. + self.assertEqual( + [], + self.connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active' AND (" + "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " + "OR json_extract(details_json,'$.rebuild_required_at_us') IS NOT NULL)", + (WORKSPACE_ID,), + ).fetchall(), + ) + + def _assert_generation_rows_collected(self) -> None: + """Every generation table keeps rows only for surviving manifests.""" + + for projection, table in _GENERATION_TABLES.items(): + rows = { + int(row[0]) + for row in self.connection.execute( + f'SELECT DISTINCT projection_generation FROM "{table}" ' + "WHERE workspace_id=?", + (WORKSPACE_ID,), + ) + } + self.assertTrue(rows, f"{table} has no rows to prove collection") + self.assertLessEqual(rows, set(self._manifests(projection)), table) def test_stale_backlog_shrinks_five_generations_per_activation(self) -> None: self._append(procedure=True) @@ -308,6 +377,146 @@ def collect_while_reading(*args, **kwargs): self.assertNotIn(oldest, self._manifests("lexical")) self.assertLessEqual(len(self._manifests("lexical")), 2) + def test_gc_in_wal_mode_collects_under_an_open_reader(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self.connection.execute("PRAGMA journal_mode=WAL") + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + table = lexical_fts_table_name(WORKSPACE_ID, oldest) + reader = sqlite3.connect(self.path) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + reader.execute("BEGIN") + reader.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + # WAL readers do not block the drop, and the reader's snapshot + # still answers from the generation being collected. + self.assertEqual([1], collected) + self.assertEqual( + 1, len(reader.execute(f'SELECT * FROM "{table}" LIMIT 1').fetchall()) + ) + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertNotIn(oldest, self._fts_generations("lexical")) + finally: + reader.close() + + def test_repair_left_rebuild_required_generations_are_collected(self) -> None: + self._append() + self._rebuild("lexical") + repaired = self._active("lexical") + # verify-v7 --repair-projections demotes the active generation and + # rebuilds in one transaction, so its GC is a no-op. + self.connection.execute( + "UPDATE projection_manifests SET status='rebuild_required' " + "WHERE workspace_id=? AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, repaired), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + self.assertNotIn(repaired, manifests) + self.assertEqual(set(manifests), self._fts_generations("lexical")) + self.assertEqual( + 0, + self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + "AND projection_generation=?", + (WORKSPACE_ID, repaired), + ).fetchone()[0], + ) + + def test_an_undroppable_generation_does_not_wedge_the_rest(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + with patch.object( + projections, "collect_superseded_generations", return_value=0 + ): + for _ in range(4): + self._rebuild("lexical") + generations = sorted(self._manifests("lexical")) + damaged = generations[0] + original = projections._drop_generation + + def refuse_oldest( + connection, workspace_id, projection, manifest_id, generation + ): + if generation == damaged: + raise sqlite3.OperationalError("malformed database schema") + return original( + connection, workspace_id, projection, manifest_id, generation + ) + + with patch.object(projections, "_drop_generation", refuse_oldest): + collected = projections.collect_superseded_generations( + self.connection, WORKSPACE_ID, "lexical" + ) + + manifests = self._manifests("lexical") + # The damaged generation is skipped, not retried forever, and every + # later victim is still collected. + self.assertGreaterEqual(collected, 1) + self.assertIn(damaged, manifests) + self.assertIn(damaged, self._fts_generations("lexical")) + self.assertNotIn(generations[1], manifests) + self.assertNotIn(generations[1], self._fts_generations("lexical")) + self.assertIsNotNone(lexical_fts_table_name(WORKSPACE_ID, damaged)) + + def test_offline_compaction_drains_a_backlog_and_reclaims_the_file(self) -> None: + from daem0nmcp.retrieval.operations import compact_projections + + self._append(procedure=True) + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(30): + for projection in ("lexical", "procedure"): + self._rebuild(projection) + self.assertEqual(30, len(self._manifests("lexical"))) + before = self.path.stat().st_size + + payload = compact_projections(self.connection, WORKSPACE_ID) + + self.assertEqual(28, payload["collected"]["lexical"]) + self.assertEqual(28, payload["collected"]["procedure"]) + self.assertTrue(payload["vacuumed"]) + self.assertEqual(58, payload["superseded_manifests_before"]) + self.assertEqual(2, payload["superseded_manifests_after"]) + for projection in ("lexical", "procedure"): + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests), projection) + self.assertEqual(set(manifests), self._fts_generations(projection)) + self.assertLess(self.path.stat().st_size, before) + def test_lock_contention_is_retried_not_dead_lettered(self) -> None: from daem0nmcp.retrieval.jobs import ProjectionJobRunner from daem0nmcp.retrieval.projections import LexicalProjectionBuilder diff --git a/tests/test_verify_v7.py b/tests/test_verify_v7.py index c50ed20..3ad9705 100644 --- a/tests/test_verify_v7.py +++ b/tests/test_verify_v7.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch import pytest @@ -198,6 +199,32 @@ def test_verify_v7_replays_every_authority_domain(tmp_path: Path) -> None: assert report["checks"]["active_pointer"]["generation"] == 1 +def test_verify_v7_reports_superseded_generation_counts(tmp_path: Path) -> None: + """A GC that has stopped making progress must be visible without logs.""" + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + storage, database = _storage(tmp_path) + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(3): + LexicalProjectionBuilder(connection).rebuild(WORKSPACE_ID) + connection.commit() + + report = verify_v7(storage, WORKSPACE_ID) + + inventory = report["checks"]["projection_generations"] + assert inventory["ok"] is True + # Three generations: one active, two superseded and uncollected. + assert inventory["lexical"] == 2 + assert inventory["outcome"] == 0 + # Only the partitions themselves, never their FTS5 shadow tables. + assert inventory["fts_partitions"] == 3 + + def test_verify_v7_accepts_intact_active_code_generation(tmp_path: Path) -> None: storage, database = _storage(tmp_path) _build_code_projection(database) From a68b2052827319e111a409a1ae8ed8393ff727fb Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 00:26:35 -0400 Subject: [PATCH 44/72] fix: honour the profile switch in health, the sandbox, and the launcher A broken-but-installed extra now degrades its own profile at startup, naming the module and exception, instead of reporting ready and failing per call; the graph preload no longer imports numpy, which the graph extra does not install. system_health and meta.capability_states carry the registry's real remediation instead of 'Review the capability profile.' DAEM0NMCP_AGENCY_E2B_ENABLED=false now stops sandbox_execute_python, which only checked E2B_API_KEY. The launcher refuses a project directory that does not exist or is the Windows system directory, keeps an exported DAEM0NMCP_PROJECT_ROOT when it is given no argument, and disables delayed expansion so a path with ! or ^ survives. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/external_operations.py | 24 ++++++ daem0nmcp/api/v7/production.py | 98 +++++++++++++++++------- daem0nmcp/capabilities.py | 5 +- start_daem0nmcp_server.bat | 35 +++++++-- start_server.py | 13 +++- tests/api_v7/test_entrypoints.py | 9 ++- tests/api_v7/test_external_operations.py | 40 ++++++++++ tests/api_v7/test_production.py | 42 +++++++++- tests/test_start_server.py | 84 ++++++++++++++++---- 9 files changed, 292 insertions(+), 58 deletions(-) diff --git a/daem0nmcp/api/v7/external_operations.py b/daem0nmcp/api/v7/external_operations.py index 57d8001..f07abcb 100644 --- a/daem0nmcp/api/v7/external_operations.py +++ b/daem0nmcp/api/v7/external_operations.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import Any, Protocol +from ...capabilities import CapabilityRegistry from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET @@ -528,6 +529,29 @@ async def sandbox_execute_python( ), ), ) + capability = CapabilityRegistry(environ=dependencies.environment).get( + "agency-e2b" + ) + if capability["status"] != "ready": + # The advertised opt-out must also switch off remote code execution. + raise ExternalOperationError( + "CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + capability_states=( + CapabilityState( + name="agency-e2b", + status=capability["status"], + reason_code="CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + remediation=( + "Install 'daem0nmcp[agency-e2b]' and leave " + "DAEM0NMCP_AGENCY_E2B_ENABLED unset or true." + ), + ), + ), + ) provider = ( dependencies.sandbox_provider_factory() if dependencies.sandbox_provider_factory diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 498cd4a..32d0bbc 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -10,7 +10,7 @@ import os import re import secrets -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from contextlib import asynccontextmanager, suppress from dataclasses import dataclass from datetime import datetime, timezone @@ -225,34 +225,46 @@ async def aclose(self) -> None: await await_projection_job_drains(tuple(self._paths)) +# Only entry points the profile's own extra installs; a module from another +# profile would fail here and degrade a working one. +_NATIVE_PROFILE_MODULES = { + "local": ("qdrant_client",), + "models-local": ("sentence_transformers", "onnx", "onnxruntime"), + "graph": ("networkx", "igraph", "leidenalg"), +} + + class _OptionalNativeRuntimeLifecycle: """Initialize enabled native provider entry points before worker threads.""" - def __init__(self, capability_statuses: Mapping[str, str]) -> None: - self._local_enabled = capability_statuses.get("local") == "ready" - self._models_enabled = capability_statuses.get("models-local") == "ready" - self._graph_enabled = capability_statuses.get("graph") == "ready" + def __init__(self, capability_statuses: MutableMapping[str, str]) -> None: + self._statuses = capability_statuses + self.failures: dict[str, str] = {} def start(self) -> None: - modules = [] - if self._local_enabled: - modules.append("qdrant_client") - if self._models_enabled: - modules.extend(("sentence_transformers", "onnx", "onnxruntime")) - if self._graph_enabled: - modules.extend(("numpy", "networkx", "igraph", "leidenalg")) - for module in modules: - # Import the public model/runtime entry points on the main thread. - # This initializes NumPy/SciPy/native DLL dependencies before a - # dense query can race another provider's first worker creation. - try: - importlib.import_module(module) - except Exception: - # Independent providers still get initialized if one is broken. - # Their operation boundary reports stable degraded diagnostics. - logging.getLogger(__name__).warning( - "Optional native provider runtime preload is unavailable" - ) + for profile, modules in _NATIVE_PROFILE_MODULES.items(): + if self._statuses.get(profile) != "ready": + continue + for module in modules: + # Import the public model/runtime entry points on the main + # thread. This initializes NumPy/SciPy/native DLL dependencies + # before a dense query can race another provider's first worker. + try: + importlib.import_module(module) + except Exception as error: + # Installed metadata said ready, but the package cannot run + # here. Report that once instead of per tool call. + self._statuses[profile] = "degraded" + self.failures[profile] = ( + f"importing {module} raised {type(error).__name__}" + ) + logging.getLogger(__name__).warning( + "Optional %s runtime is unavailable: importing %s raised %s", + profile, + module, + type(error).__name__, + ) + break def _loopback_host(host: str) -> bool: @@ -304,29 +316,56 @@ def _task_configuration( ) +def _remediation_text(name: str, remediation: Mapping[str, Any]) -> str: + """Render the registry's structured remediation as one actionable line.""" + + parts = [ + str(remediation[key]) + for key in ("message", "command", "environment") + if remediation.get(key) + ] + missing = remediation.get("missing") + if isinstance(missing, list) and missing: + parts.append("Missing: " + ", ".join(str(item) for item in missing[:8]) + ".") + return " ".join(parts) or f"Review the {name} capability profile." + + def _capability_states( environ: Mapping[str, str], + native_failures: Mapping[str, str] = MappingProxyType({}), ) -> tuple[CapabilityState, ...]: values: list[CapabilityState] = [] for name, capability in CapabilityRegistry(environ=environ).all().items(): status = str(capability["status"]) - if status == "ready": + failure = native_failures.get(name) + if status == "ready" and failure is None: values.append( CapabilityState.model_validate({"name": name, "status": "ready"}) ) continue + if failure is not None: + status = "degraded" reason = { "disabled": "CAPABILITY_DISABLED", "degraded": "CAPABILITY_DEGRADED", "failed": "CAPABILITY_CONFIGURATION_INVALID", }[status] + remediation = capability.get("remediation") + text = ( + f"The {name} profile is installed but cannot run here: {failure}. " + "Reinstall or repair it." + if failure is not None + else _remediation_text( + name, remediation if isinstance(remediation, Mapping) else {} + ) + ) values.append( CapabilityState.model_validate( { "name": name, "status": status, "reason_code": reason, - "remediation": f"Review the {name} capability profile.", + "remediation": text, } ) ) @@ -709,6 +748,10 @@ def _assemble( name: str(capability["status"]) for name, capability in CapabilityRegistry(environ=env).all().items() } + # Preload before anything snapshots the statuses, so an installed but + # broken extra degrades everywhere instead of only failing per call. + native_runtimes = _OptionalNativeRuntimeLifecycle(capability_statuses) + native_runtimes.start() normalizer = build_argument_normalizer() registry = WorkspaceRegistry.from_settings(loaded_settings) workspaces = {registry.default.workspace_id: registry.default} @@ -837,7 +880,7 @@ def schedule_capture_projection(path: Path) -> None: else "jwt" ), task_support=task_state, - capability_states=_capability_states(env), + capability_states=_capability_states(env, native_runtimes.failures), storage_resolver=storage_resolver, dreaming_provider=dreaming.health, runtime_diagnostics_provider=runtime_health.inspect, @@ -983,7 +1026,6 @@ def authorize_bridge_workspace( edit_bridge_service=edit_bridge_service, services=( WorkspaceBootstrapLifecycle(tuple(workspaces.values())), - _OptionalNativeRuntimeLifecycle(capability_statuses), writer, recall, runtime_health, diff --git a/daem0nmcp/capabilities.py b/daem0nmcp/capabilities.py index 36f022b..71e8a5e 100644 --- a/daem0nmcp/capabilities.py +++ b/daem0nmcp/capabilities.py @@ -252,7 +252,10 @@ def _failed(self, profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "fix_configuration", "environment": profile.environment_key, - "message": "Use a boolean value: true or false.", + "message": ( + "Use true or false, or leave it unset to enable the " + "profile automatically when its extra is installed." + ), }, } diff --git a/start_daem0nmcp_server.bat b/start_daem0nmcp_server.bat index e2a0704..08214d1 100644 --- a/start_daem0nmcp_server.bat +++ b/start_daem0nmcp_server.bat @@ -6,17 +6,23 @@ REM This script starts the Daem0nMCP HTTP server REM for one project on Windows. REM REM Usage: start_daem0nmcp_server.bat [project_dir] -REM The project defaults to the directory you run -REM it from. Stdio (python -m daem0nmcp) also works -REM on Windows; use this launcher when you want an -REM HTTP server instead. +REM The project defaults to an exported +REM DAEM0NMCP_PROJECT_ROOT, else the directory you +REM run it from. Stdio (python -m daem0nmcp) also +REM works on Windows; use this launcher when you +REM want an HTTP server instead. REM ============================================ title Daem0nMCP Server -setlocal +REM Delayed expansion would re-parse ! and ^ in a project path +setlocal DisableDelayedExpansion REM Capture the project before changing directory -if "%~1"=="" (set "DAEM0N_PROJECT=%CD%") else (set "DAEM0N_PROJECT=%~f1") +set "DAEM0N_PROJECT=" +if not "%~1"=="" set "DAEM0N_PROJECT=%~f1" +if "%~1"=="" if not defined DAEM0NMCP_PROJECT_ROOT set "DAEM0N_PROJECT=%CD%" +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%" goto :system_directory +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%\System32" goto :system_directory REM Change to the script's directory cd /d "%~dp0" @@ -32,9 +38,13 @@ echo \^|/ echo * echo. -REM Start the server +REM Start the server. An exported DAEM0NMCP_PROJECT_ROOT is left alone. REM The trailing "\." keeps a drive root such as C:\ from escaping the quote -python start_server.py --port 9876 --project "%DAEM0N_PROJECT%\." +if defined DAEM0N_PROJECT ( + python start_server.py --port 9876 --project "%DAEM0N_PROJECT%\." +) else ( + python start_server.py --port 9876 +) REM If the server exits, pause so user can see any errors if errorlevel 1 ( @@ -42,3 +52,12 @@ if errorlevel 1 ( echo [ERROR] Server exited with an error. Press any key to close. pause >nul ) +goto :eof + +:system_directory +echo. +echo [ERROR] Refusing to serve "%DAEM0N_PROJECT%". +echo Run the launcher from your project, or pass it: +echo start_daem0nmcp_server.bat C:\path\to\project +pause >nul +exit /b 2 diff --git a/start_server.py b/start_server.py index 54cbcc7..dd5f5cf 100644 --- a/start_server.py +++ b/start_server.py @@ -29,10 +29,15 @@ def main(): ) args = parser.parse_args() - if args.project: - os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(Path(args.project).resolve()) - elif not os.environ.get("DAEM0NMCP_PROJECT_ROOT"): - os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(Path.cwd().resolve()) + project_root = Path( + args.project or os.environ.get("DAEM0NMCP_PROJECT_ROOT") or Path.cwd() + ) + if not project_root.is_dir(): + # Otherwise the server silently creates and serves an empty workspace + # wherever a typo points. + parser.error(f"project directory does not exist: {project_root}") + os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root.resolve()) + print(f"Serving project: {project_root.resolve()}", file=sys.stderr) # Import only after the workspace environment is fixed. Both public # launchers use the same v7 composition and transport-security boundary. diff --git a/tests/api_v7/test_entrypoints.py b/tests/api_v7/test_entrypoints.py index dcd0b15..fdad6c8 100644 --- a/tests/api_v7/test_entrypoints.py +++ b/tests/api_v7/test_entrypoints.py @@ -2,7 +2,9 @@ import io import os +import shutil import sys +import tempfile import types import unittest from contextlib import redirect_stdout @@ -15,6 +17,9 @@ def test_http_launcher_builds_remote_v7_server_and_uses_shared_runner(self) -> N import start_server from daem0nmcp.api.v7.launcher import ServerOptions + workspace = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, workspace, True) + built: list[tuple[str, str | None]] = [] launched: list[tuple[object, ServerOptions]] = [] server = object() @@ -26,7 +31,7 @@ def create_server( ) -> object: self.assertEqual( os.environ.get("DAEM0NMCP_PROJECT_ROOT"), - str(Path("tests/http-workspace").resolve()), + str(workspace.resolve()), ) built.append((transport, host)) return server @@ -40,7 +45,7 @@ def create_server( "--port", "9988", "--project", - str(Path("tests/http-workspace")), + str(workspace), ] output = io.StringIO() with ( diff --git a/tests/api_v7/test_external_operations.py b/tests/api_v7/test_external_operations.py index 1a1126d..eda7561 100644 --- a/tests/api_v7/test_external_operations.py +++ b/tests/api_v7/test_external_operations.py @@ -737,6 +737,46 @@ async def test_missing_api_key_keeps_operation_registered_but_disabled( "Set E2B_API_KEY to enable isolated Python execution.", ) + async def test_disabling_the_profile_switches_remote_execution_off(self) -> None: + """The advertised opt-out must also stop the sandbox.""" + from daem0nmcp.api.v7.external_operations import ( + ExternalOperationDependencies, + ExternalOperationError, + build_external_operations, + ) + from daem0nmcp.api.v7.record_operations import RecordOperationDependencies + from daem0nmcp.workspace import WorkspaceRegistry + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + workspace = WorkspaceRegistry([root], default_root=root).default + operations = build_external_operations( + ExternalOperationDependencies( + record_dependencies=RecordOperationDependencies(), + environment={ + "E2B_API_KEY": "key", + "DAEM0NMCP_AGENCY_E2B_ENABLED": "false", + }, + sandbox_provider_factory=lambda: self.fail("provider was built"), + ) + ) + with self.assertRaises(ExternalOperationError) as caught: + await operations["sandbox_execute_python"]( + workspace=workspace, + request=_request( + "sandbox_execute_python", + workspace_id=workspace.workspace_id, + code="print(1)", + timeout_seconds=1, + preflight_token=PREFLIGHT_TOKEN, + ), + ) + + self.assertEqual(caught.exception.code, "CAPABILITY_DISABLED") + state = caught.exception.capability_states[0] + self.assertEqual(state.name, "agency-e2b") + self.assertIn("DAEM0NMCP_AGENCY_E2B_ENABLED", str(state.remediation)) + if __name__ == "__main__": unittest.main() diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index c6f1ac2..e9eba9b 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -98,7 +98,7 @@ def test_native_runtime_preloads_only_each_enabled_profile( "sentence_transformers", "onnx", "onnxruntime", - "numpy", + # numpy belongs to models-local; the graph extra never installs it. "networkx", "igraph", "leidenalg", @@ -106,6 +106,46 @@ def test_native_runtime_preloads_only_each_enabled_profile( imported, ) + def test_a_broken_installed_runtime_degrades_its_own_profile_only(self) -> None: + """Metadata can say ready while the package cannot import here.""" + from daem0nmcp.api.v7 import production + + def import_module(name: str) -> object: + if name == "igraph": + raise OSError("[WinError 126] The specified module could not be found") + return object() + + statuses = {"graph": "ready", "local": "ready"} + with patch.object(production.importlib, "import_module", import_module): + lifecycle = production._OptionalNativeRuntimeLifecycle(statuses) + lifecycle.start() + + self.assertEqual({"graph": "degraded", "local": "ready"}, statuses) + self.assertIn("igraph", lifecycle.failures["graph"]) + self.assertIn("OSError", lifecycle.failures["graph"]) + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "true"}, lifecycle.failures + ) + } + self.assertEqual("degraded", states["graph"].status) + self.assertIn("igraph", str(states["graph"].remediation)) + + def test_capability_states_carry_the_registry_remediation(self) -> None: + """Health must say how to enable a profile, not 'review' it.""" + from daem0nmcp.api.v7 import production + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "false"} + ) + } + self.assertEqual("disabled", states["graph"].status) + self.assertIn("DAEM0NMCP_GRAPH_ENABLED", str(states["graph"].remediation)) + def test_full_manifest_rejects_missing_or_unexpected_resources(self) -> None: from daem0nmcp.api.v7.production import build_production_surface from daem0nmcp.api.v7.registry import ManifestError diff --git a/tests/test_start_server.py b/tests/test_start_server.py index 0426467..f53604b 100644 --- a/tests/test_start_server.py +++ b/tests/test_start_server.py @@ -40,12 +40,16 @@ def create_server(transport: str, *, host: str | None = None) -> object: def test_exported_project_root_survives(tmp_path): - exported = str(tmp_path / "exported") - assert _launched_project_root([], {"DAEM0NMCP_PROJECT_ROOT": exported}) == exported + exported = tmp_path / "exported" + exported.mkdir() + root = _launched_project_root([], {"DAEM0NMCP_PROJECT_ROOT": str(exported)}) + assert root == str(exported.resolve()) def test_project_argument_wins_over_exported_root(tmp_path): chosen = tmp_path / "chosen" + chosen.mkdir() + (tmp_path / "exported").mkdir() root = _launched_project_root( ["--project", str(chosen)], {"DAEM0NMCP_PROJECT_ROOT": str(tmp_path / "exported")}, @@ -58,29 +62,81 @@ def test_current_directory_is_the_default(tmp_path, monkeypatch): assert _launched_project_root([], {}) == str(tmp_path.resolve()) +def test_a_missing_project_directory_is_refused(tmp_path): + """A typo must not create and serve an empty workspace.""" + with pytest.raises(SystemExit) as raised: + _launched_project_root(["--project", str(tmp_path / "typo")], {}) + assert raised.value.code == 2 + + @pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") @pytest.mark.parametrize("use_argument", [False, True]) def test_batch_launcher_passes_the_callers_project(tmp_path, use_argument): """The .bat changes to the repo dir, so it must capture the project first.""" + recorded, environment = _shim(tmp_path) + project = tmp_path / "my project (x86) ^ !name!" + project.mkdir() + + _run_batch( + ["my project (x86) ^ !name!"] if use_argument else [], + cwd=tmp_path if use_argument else project, + environment=environment, + ) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == f'start_server.py --port 9876 --project "{project}\\."' + assert Path(f"{project}\\.").resolve() == project.resolve() + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_leaves_an_exported_project_root_alone(tmp_path): + recorded, environment = _shim(tmp_path) + exported = tmp_path / "exported" + exported.mkdir() + environment["DAEM0NMCP_PROJECT_ROOT"] = str(exported) + + _run_batch([], cwd=tmp_path, environment=environment) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == "start_server.py --port 9876" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_refuses_the_windows_system_directory(tmp_path): + """ "Run as administrator" starts in System32; that is never the project.""" + recorded, environment = _shim(tmp_path) + system32 = Path(os.environ["SYSTEMROOT"]) / "System32" + + completed = _run_batch([], cwd=system32, environment=environment, check=False) + + assert completed.returncode == 2 + assert not recorded.exists() + + +def _shim(tmp_path: Path) -> tuple[Path, dict[str, str]]: + """Put a python.cmd on PATH that records the arguments it is given.""" shim = tmp_path / "shim" shim.mkdir() recorded = tmp_path / "argv.txt" (shim / "python.cmd").write_text(f'@echo %*> "{recorded}"\r\n', encoding="ascii") - project = tmp_path / "my project (x86)" - project.mkdir() environment = {**os.environ, "PATH": f"{shim}{os.pathsep}{os.environ['PATH']}"} - - subprocess.run( - ["cmd", "/c", str(REPO_ROOT / "start_daem0nmcp_server.bat")] - + (["my project (x86)"] if use_argument else []), - cwd=tmp_path if use_argument else project, + environment.pop("DAEM0NMCP_PROJECT_ROOT", None) + return recorded, environment + + +def _run_batch( + arguments: list[str], + *, + cwd: Path, + environment: dict[str, str], + check: bool = True, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["cmd", "/c", str(REPO_ROOT / "start_daem0nmcp_server.bat"), *arguments], + cwd=cwd, env=environment, stdin=subprocess.DEVNULL, capture_output=True, - check=True, + check=check, timeout=30, ) - - arguments = recorded.read_text(encoding="ascii").strip() - assert arguments == f'start_server.py --port 9876 --project "{project}\\."' - assert Path(f"{project}\\.").resolve() == project.resolve() From 7df3bc27a37a989b0ee8fd04e26682e0bf251712 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 00:29:30 -0400 Subject: [PATCH 45/72] fix: keep the OpenCode installer inside the project, and cover the new states _ensure_file refuses a target reached through a symlink, creates files exclusively, and replaces rather than writes through one on --force. Tests cover empty = auto, that a disabled profile is never probed, an installed extra reaching ready in a real server with no variable set, each apps remediation, the packaged command files, and --force/--dry-run. The CHANGELOG records the behaviour change for existing installs. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 21 ++++++++ daem0nmcp/opencode_install.py | 37 ++++++++++--- tests/api_v7/test_discovery_operations.py | 24 +++++++++ tests/api_v7/test_process_client.py | 19 +++++++ tests/test_capabilities.py | 25 ++++++++- tests/test_opencode_install.py | 64 +++++++++++++++++++++-- 6 files changed, 177 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c67c0..2a1f23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ ### Added - Reproducible v7 core foundation with opt-in dependency profiles and lazy capability reporting. +### Changed +- **Behaviour change — installed profiles now turn themselves on.** An unset (or + empty) `DAEM0NMCP__ENABLED` means "auto": the profile is ready when + its extra is installed and disabled otherwise. `false` still turns it off, and + `true` with packages missing is still degraded. An empty value used to mean + `false`. What this costs if you installed an extra and never set the flag: + `[graph]` imports networkx, igraph and leidenalg at startup, adds a graph + provider to recall (so rankings change) and lets dreaming refresh communities; + `[local]`/`[models-local]` import sentence-transformers and onnxruntime at + startup, download the embedding model on first use, and rebuild the dense + projection, which re-embeds the whole workspace once. Opt out per profile with + `DAEM0NMCP__ENABLED=false`. +- `sandbox_execute_python` now honours `DAEM0NMCP_AGENCY_E2B_ENABLED=false`; it + previously required only `E2B_API_KEY`. +- `system_health` and `meta.capability_states` report the profile's real + remediation (install command or environment variable) instead of "Review the + <name> capability profile." +- `start_daem0nmcp_server.bat` serves the directory you run it from (or its first + argument, or an exported `DAEM0NMCP_PROJECT_ROOT`) instead of the Daem0n-MCP + checkout, and `install-opencode` installs the four `.opencode/commands/` files. + ## [6.0.0] - 2026-01-29 ### Breaking Changes diff --git a/daem0nmcp/opencode_install.py b/daem0nmcp/opencode_install.py index 36d9af4..98f05b5 100644 --- a/daem0nmcp/opencode_install.py +++ b/daem0nmcp/opencode_install.py @@ -98,20 +98,45 @@ def _ensure_dir(path: Path, dry_run: bool) -> str: return "[create]" -def _ensure_file(path: Path, content: str, dry_run: bool, force: bool) -> str: +def _reject_links(path: Path, root: Path) -> None: + """Refuse a target that a link could move outside the project.""" + current = path + while True: + if current.is_symlink(): + raise OSError(f"refusing to write through the link {current}") + if current == root: + return + parent = current.parent + if parent == current: + raise OSError(f"{path} is outside {root}") + current = parent + + +def _ensure_file( + path: Path, content: str, dry_run: bool, force: bool, root: Path +) -> str: """Ensure a file exists with given content. Returns status string.""" + _reject_links(path, root) if path.exists(): if force: if not dry_run: - path.write_text(content, encoding="utf-8") + # Replace rather than write in place, so the new file is ours. + path.unlink() + _write_new(path, content) return "[overwrite]" return "[exists]" if not dry_run: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") + _reject_links(path.parent, root) + _write_new(path, content) return "[create]" +def _write_new(path: Path, content: str) -> None: + with path.open("x", encoding="utf-8") as handle: + handle.write(content) + + def _configured_template( installation: BridgeInstallation | None, ) -> dict[str, Any]: @@ -268,7 +293,7 @@ def install_opencode( lines.append("Configuration:") json_content = json.dumps(_configured_template(installation), indent=2) + "\n" json_path = root / "opencode.json" - status = _ensure_file(json_path, json_content, dry_run, force) + status = _ensure_file(json_path, json_content, dry_run, force, root) lines.append(f" {status} opencode.json") if ( installation is not None @@ -296,13 +321,13 @@ def install_opencode( # -- Ensure plugin file ------------------------------------------ plugin_path = opencode_root / "plugins" / "daem0n.ts" - status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force) + status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force, root) lines.append(f" {status} .opencode/plugins/daem0n.ts") # -- Ensure slash-command files ---------------------------------- for name, content in sorted(COMMAND_TEMPLATES.items()): status = _ensure_file( - opencode_root / "commands" / name, content, dry_run, force + opencode_root / "commands" / name, content, dry_run, force, root ) lines.append(f" {status} .opencode/commands/{name}") diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index a40b2d8..3c00bfe 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1586,6 +1586,30 @@ def factory(): self.assertFalse(loaded) self.assertEqual("CAPABILITY_DISABLED", raised.exception.code) self.assertEqual("apps", raised.exception.capability_states[0].name) + self.assertIn( + "daem0nmcp[apps]", raised.exception.capability_states[0].remediation + ) + + async def test_each_apps_status_names_its_own_remedy(self) -> None: + """An invalid setting is a configuration fix, not a missing package.""" + from daem0nmcp.api.v7.discovery_operations import _code_capability_error + + remedies = {} + for status in ("disabled", "degraded", "failed"): + error = _code_capability_error( + SimpleNamespace(capability_statuses={"apps": status}) + ) + assert error is not None + remedies[status] = error.capability_states[0].remediation + + self.assertIn("unless DAEM0NMCP_APPS_ENABLED=false", remedies["disabled"]) + self.assertEqual(remedies["degraded"], remedies["disabled"]) + self.assertIn("true, false, or leave it unset", remedies["failed"]) + self.assertIsNone( + _code_capability_error( + SimpleNamespace(capability_statuses={"apps": "ready"}) + ) + ) async def test_code_index_source_change_rolls_back_before_publish(self) -> None: from daem0nmcp.api.v7.discovery_operations import ( diff --git a/tests/api_v7/test_process_client.py b/tests/api_v7/test_process_client.py index 8f71340..98cad9f 100644 --- a/tests/api_v7/test_process_client.py +++ b/tests/api_v7/test_process_client.py @@ -452,3 +452,22 @@ async def test_consolidation_preview_copy_and_canonical_archive_over_real_mcp( ) assert archived["archived"] == 2 assert all(source.root.is_dir() for source in sources) + + +async def test_an_installed_extra_is_enabled_without_any_variable( + initialized_workspace, +): + """UD-7: installing the extra is the whole opt-in.""" + from daem0nmcp.capabilities import CapabilityRegistry + + if CapabilityRegistry(environ={}).get("graph")["status"] != "ready": + pytest.skip("the graph extra is not installed") + + workspace = initialized_workspace + scope = {"workspace_id": workspace.workspace_id} + async with process_client(workspace.root, "stdio") as session: + await succeed(session, "session_brief", scope) + health = await succeed(session, "system_health", scope) + + states = {state["name"]: state["status"] for state in health["capability_states"]} + assert states["graph"] == "ready", states diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index dfeb94a..ef5712c 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -246,13 +246,33 @@ def test_unset_variable_leaves_a_missing_extra_disabled(self): }, ) + def test_an_empty_value_is_treated_as_unset(self): + """An empty variable means auto, as an unset one does.""" + from daem0nmcp.capabilities import CapabilityRegistry + + installed = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": " "}, + module_available=lambda _: True, + ).get("graph") + missing = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": ""}, + module_available=lambda _: False, + ).get("graph") + + self.assertEqual(installed["status"], "ready") + self.assertEqual(missing["status"], "disabled") + self.assertEqual(missing["remediation"]["action"], "install_extra") + def test_explicit_false_disables_an_installed_profile(self): - """The variable still turns an installed profile off.""" + """The variable still turns an installed profile off, without probing.""" from daem0nmcp.capabilities import CapabilityRegistry + def probe(distribution): + raise AssertionError(f"probed {distribution} for a disabled profile") + capability = CapabilityRegistry( environ={"DAEM0NMCP_GRAPH_ENABLED": "false"}, - module_available=lambda _: True, + module_available=probe, ).get("graph") self.assertEqual(capability["status"], "disabled") @@ -303,6 +323,7 @@ def test_invalid_enablement_value_is_failed_with_structured_remediation(self): self.assertEqual( capability["remediation"]["environment"], "DAEM0NMCP_MODELS_LOCAL_ENABLED" ) + self.assertIn("unset", capability["remediation"]["message"]) def test_ready_profile_checks_only_the_requested_profile(self): """Capability inspection must not import or probe every optional subsystem.""" diff --git a/tests/test_opencode_install.py b/tests/test_opencode_install.py index d9d7e50..8b0c0c6 100644 --- a/tests/test_opencode_install.py +++ b/tests/test_opencode_install.py @@ -11,6 +11,7 @@ from daem0nmcp.edit_bridge_transport import provision_bridge_credential from daem0nmcp.opencode_install import ( + COMMAND_TEMPLATES, PLUGIN_TEMPLATE, install_opencode, select_opencode_interface, @@ -232,11 +233,7 @@ def test_install_preserves_existing_opencode_json(tmp_path): def test_install_creates_repo_command_files_without_overwriting(tmp_path): """The installer ships the repo's slash commands and keeps user edits.""" - repo_commands = Path(__file__).resolve().parents[1] / ".opencode" / "commands" - expected = { - path.name: path.read_text(encoding="utf-8") - for path in repo_commands.glob("*.md") - } + expected = _repo_commands() assert set(expected) == {"commune.md", "counsel.md", "inscribe.md", "recall.md"} commands = tmp_path / ".opencode" / "commands" commands.mkdir(parents=True) @@ -252,3 +249,60 @@ def test_install_creates_repo_command_files_without_overwriting(tmp_path): assert (commands / "recall.md").read_text(encoding="utf-8") == "my recall" for name in ("commune.md", "counsel.md", "inscribe.md"): assert (commands / name).read_text(encoding="utf-8") == expected[name] + + +def _repo_commands() -> dict[str, str]: + repo_commands = Path(__file__).resolve().parents[1] / ".opencode" / "commands" + return { + path.name: path.read_text(encoding="utf-8") + for path in repo_commands.glob("*.md") + } + + +def test_packaged_commands_match_the_repo_commands(): + """The shipped copies must not drift from the ones the repo uses.""" + assert _repo_commands() == COMMAND_TEMPLATES + + +def test_force_overwrites_commands_and_dry_run_writes_nothing(tmp_path): + ok, msg = install_opencode( + str(tmp_path), dry_run=True, bridge_config_root=tmp_path / "host-config" + ) + assert ok, msg + assert "[create] .opencode/commands/recall.md" in msg + assert not (tmp_path / ".opencode" / "commands" / "recall.md").exists() + + assert install_opencode(str(tmp_path), bridge_config_root=tmp_path / "host-config")[ + 0 + ] + recall = tmp_path / ".opencode" / "commands" / "recall.md" + recall.write_text("edited", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), force=True, bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[overwrite] .opencode/commands/recall.md" in msg + assert recall.read_text(encoding="utf-8") == COMMAND_TEMPLATES["recall.md"] + + +def test_installer_refuses_to_write_through_a_symlink(tmp_path): + """A cloned repo must not steer the installer outside the project.""" + outside = tmp_path / "outside" + outside.mkdir() + project = tmp_path / "project" + (project / ".opencode" / "commands").mkdir(parents=True) + link = project / ".opencode" / "commands" / "recall.md" + try: + link.symlink_to(outside / "stolen.md") + except (OSError, NotImplementedError): + pytest.skip("symlinks are unavailable") + + ok, msg = install_opencode( + str(project), bridge_config_root=tmp_path / "host-config" + ) + + assert not ok + assert "link" in msg + assert not (outside / "stolen.md").exists() From 5c72fc39b874e6fdff1aad7ad6cceaf171c01d43 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:32:43 -0400 Subject: [PATCH 46/72] test: show the server's own log when a process test fails CI shows only the assertion, so an INTERNAL_ERROR or a crash inside the server is invisible: the tmp directory with stdio-server.log and http-server.log is gone by then. The harness now prints the log tail when the body fails. The portability import helper also retries a contended memory_preflight, which a client must do now that it is retryable, and bounds the attempts so a transfer that always contends fails instead of crawling. Co-Authored-By: Claude Opus 5 --- tests/api_v7/process_client.py | 32 +++++++++++++++++ tests/api_v7/test_process_portability.py | 44 +++++++++++++++++------- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index e31e2f9..9f69f15 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -150,6 +150,29 @@ def server_environment( return environment +_LOG_TAIL_BYTES = 8000 + + +def dump_server_log(path: Path) -> None: + """Print the server's own log when a process test fails. + + CI shows only the assertion, so an INTERNAL_ERROR or a crash inside the + server is otherwise invisible; the tmp directory is gone by then. + """ + + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError as error: + print(f"\n----- {path.name} unavailable: {error} -----", flush=True) + return + tail = text[-_LOG_TAIL_BYTES:] + print( + f"\n----- {path.name} (last {len(tail)} of {len(text)} chars) -----\n{tail}" + f"\n----- end {path.name} -----", + flush=True, + ) + + @asynccontextmanager async def process_client( workspace: Path, @@ -192,10 +215,12 @@ async def process_client( # SDK stdout-reader BrokenResourceError during shutdown. body_error = error except BaseException as cleanup_error: + dump_server_log(workspace / log_name) if body_error is not None: raise body_error from cleanup_error raise if body_error is not None: + dump_server_log(workspace / log_name) raise body_error return if transport != "streamable-http": @@ -203,6 +228,7 @@ async def process_client( with socket.socket() as reservation: reservation.bind(("127.0.0.1", 0)) port = reservation.getsockname()[1] + failed = False with (workspace / "http-server.log").open("w", encoding="utf-8") as log: process = subprocess.Popen( [ @@ -257,6 +283,9 @@ async def process_client( ): await session.initialize() yield session + except BaseException: + failed = True + raise finally: if process.poll() is None: process.terminate() @@ -265,6 +294,9 @@ async def process_client( except subprocess.TimeoutExpired: process.kill() await asyncio.to_thread(process.wait, timeout=10) + if failed: + log.flush() + dump_server_log(workspace / "http-server.log") async def call(session: ClientSession, tool: str, arguments: dict) -> dict: diff --git a/tests/api_v7/test_process_portability.py b/tests/api_v7/test_process_portability.py index ee6198d..f86cd8c 100644 --- a/tests/api_v7/test_process_portability.py +++ b/tests/api_v7/test_process_portability.py @@ -45,28 +45,46 @@ async def _import_retrying_while_busy(session, scope, arguments): """Import one page, waiting out a concurrent writer's SQLite write lock. A projection drain or dreaming job can hold the write lock past the server's - busy timeout, which the transfer reports as the retryable DATABASE_IN_USE. - A real client retries it; so does this test, within a bounded deadline. + busy timeout, which both the preflight and the transfer report as the + retryable DATABASE_IN_USE. A real client retries those; so does this test, + within a bounded deadline and a bounded number of attempts. """ deadline = asyncio.get_running_loop().time() + 30 - while True: - result = await call( + for attempt in range(8): + contended = None + preflight = await call( session, - "workspace_import", + "memory_preflight", { **scope, - **arguments, - "preflight_token": await _preflight( - session, scope, "workspace_import", arguments - ), + "target_tool": "workspace_import", + "target_arguments": arguments, + "description": "Exercise workspace_import through the transport", }, ) - if result["ok"]: - return result["data"] - assert result["error"]["code"] == "DATABASE_IN_USE", result["error"] - assert asyncio.get_running_loop().time() < deadline, result["error"] + if preflight["ok"]: + result = await call( + session, + "workspace_import", + { + **scope, + **arguments, + "preflight_token": preflight["data"]["preflight_token"], + }, + ) + if result["ok"]: + # Contention is occasional; a transfer that always contends is + # a regression, not slowness. + assert attempt < 4, f"import needed {attempt + 1} attempts" + return result["data"] + contended = result["error"] + else: + contended = preflight["error"] + assert contended["code"] == "DATABASE_IN_USE", contended + assert asyncio.get_running_loop().time() < deadline, contended await asyncio.sleep(0.5) + raise AssertionError("the import stayed contended for 8 attempts") @pytest.mark.parametrize("transport", ["stdio", "streamable-http"]) From 4a50c0491252c0278156149bc76a233d84aa89f8 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:37:43 -0400 Subject: [PATCH 47/72] test: dump the http server log after its file is closed Co-Authored-By: Claude Opus 5 --- tests/api_v7/process_client.py | 138 +++++++++++++++++---------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index 9f69f15..4dca20a 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -229,74 +229,78 @@ async def process_client( reservation.bind(("127.0.0.1", 0)) port = reservation.getsockname()[1] failed = False - with (workspace / "http-server.log").open("w", encoding="utf-8") as log: - process = subprocess.Popen( - [ - python, - "-m", - "daem0nmcp.server", - "--transport", - transport, - "--host", - "127.0.0.1", - "--port", - str(port), - ], - cwd=workspace, - env=environment, - stdout=log, - stderr=log, - creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0, - ) - try: - deadline = asyncio.get_running_loop().time() + 30 - while True: - if process.poll() is not None: - raise RuntimeError( - f"MCP server exited with code {process.returncode}; inspect http-server.log" - ) - try: - _, writer = await asyncio.open_connection("127.0.0.1", port) - writer.close() - await writer.wait_closed() - break - except OSError: - if asyncio.get_running_loop().time() >= deadline: - raise TimeoutError( - "MCP HTTP server did not become ready" - ) from None - await asyncio.sleep(0.05) - if http_probe is not None: - await http_probe(f"http://127.0.0.1:{port}/mcp") - async with ( - httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, - streamable_http_client( - f"http://127.0.0.1:{port}/mcp", http_client=http_client - ) as ( - read, - write, - _, - ), - ClientSession( - read, write, read_timeout_seconds=timedelta(seconds=30) - ) as session, - ): - await session.initialize() - yield session - except BaseException: - failed = True - raise - finally: - if process.poll() is None: - process.terminate() + try: + with (workspace / "http-server.log").open("w", encoding="utf-8") as log: + process = subprocess.Popen( + [ + python, + "-m", + "daem0nmcp.server", + "--transport", + transport, + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=workspace, + env=environment, + stdout=log, + stderr=log, + creationflags=subprocess.CREATE_NO_WINDOW + if sys.platform == "win32" + else 0, + ) try: - await asyncio.to_thread(process.wait, timeout=10) - except subprocess.TimeoutExpired: - process.kill() - await asyncio.to_thread(process.wait, timeout=10) - if failed: - log.flush() - dump_server_log(workspace / "http-server.log") + deadline = asyncio.get_running_loop().time() + 30 + while True: + if process.poll() is not None: + raise RuntimeError( + f"MCP server exited with code {process.returncode}; inspect http-server.log" + ) + try: + _, writer = await asyncio.open_connection("127.0.0.1", port) + writer.close() + await writer.wait_closed() + break + except OSError: + if asyncio.get_running_loop().time() >= deadline: + raise TimeoutError( + "MCP HTTP server did not become ready" + ) from None + await asyncio.sleep(0.05) + if http_probe is not None: + await http_probe(f"http://127.0.0.1:{port}/mcp") + async with ( + httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, + streamable_http_client( + f"http://127.0.0.1:{port}/mcp", http_client=http_client + ) as ( + read, + write, + _, + ), + ClientSession( + read, write, read_timeout_seconds=timedelta(seconds=30) + ) as session, + ): + await session.initialize() + yield session + except BaseException: + failed = True + raise + finally: + if process.poll() is None: + process.terminate() + try: + await asyncio.to_thread(process.wait, timeout=10) + except subprocess.TimeoutExpired: + process.kill() + await asyncio.to_thread(process.wait, timeout=10) + finally: + if failed: + # Outside the log context, so the tail is flushed and complete. + dump_server_log(workspace / "http-server.log") async def call(session: ClientSession, tool: str, arguments: dict) -> dict: From 1783c07f0f88e870c56c7b8eb7b2f8b814973ed2 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:46:59 -0400 Subject: [PATCH 48/72] fix: brief a migrated workspace instead of refusing its table snapshots `migrate-v7` records a projection manifest for each table it fills directly (`memory_records`, `memory_fact_versions`, `memory_relationship_versions`, `entities`, `communities`), and the briefing reader refused every manifest name outside the retrieval set. The error became RESOURCE_REPOSITORY_UNAVAILABLE, then BRIEFING_FAILED, then INTERNAL_ERROR, so `session_brief` failed on every migrated v6 workspace and the covenant then blocked every other tool. `RETRIEVAL_PROJECTION_NAMES` now names the retrieval projections once. The freshness reader selects them in SQL rather than raising on the rest, so an already-migrated database briefs with no data change, and `verification_v7` derives its supported set from the same constant plus the table snapshots it owns. Those snapshots also stop carrying the live cursor and row-count invariants: nothing rebuilds them, so any later write moved a migrated store permanently to `verify-v7: invalid`. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/resource_repository.py | 26 +++++++++++++------------ daem0nmcp/retrieval/lexical_config.py | 8 ++++++++ daem0nmcp/verification_v7.py | 25 ++++++++++++++++++------ 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index 30f5ebc..1bb3d04 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -27,10 +27,11 @@ from typing import TypeVar, cast from ...bounded_workers import BoundedWorkerPool +from ...retrieval.lexical_config import RETRIEVAL_PROJECTION_NAMES from ...schema_version import CURRENT_SCHEMA_VERSION from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path -from .models import RecordSummary +from .models import RecordSummary, stored_relative_path from .public_ids import PublicObjectIdRepository from .resources import ( RESOURCE_FETCH_LIMIT, @@ -182,9 +183,9 @@ def close(self) -> None: _MAX_GIT_OUTPUT_BYTES = 1_048_576 _MAX_GIT_CHANGES = 200 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") -_PROJECTION_NAMES = frozenset( - {"lexical", "dense", "graph", "temporal", "procedure", "outcome", "code"} -) +_PROJECTION_NAMES = RETRIEVAL_PROJECTION_NAMES +_PROJECTION_NAME_PLACEHOLDERS = ",".join("?" * len(_PROJECTION_NAMES)) +_PROJECTION_NAME_PARAMETERS = tuple(sorted(_PROJECTION_NAMES)) _REQUIRED_TABLES = frozenset( { "active_context", @@ -767,12 +768,17 @@ def _read_projection_freshness_sync( workspace: Workspace, connection: sqlite3.Connection, ) -> tuple[list[object], int]: + # ``projection_manifests`` also holds the migration's table snapshots + # (``memory_records`` and friends), which no retrieval provider serves. + # Select the retrieval names instead of refusing the rest: a migrated + # workspace must brief without a data change. rows = connection.execute( "SELECT projection_name,generation,source_event_root_hash," "COALESCE(activated_at_us,completed_at_us,started_at_us) AS built_at_us " "FROM projection_manifests WHERE workspace_id=? AND status='active' " + f"AND projection_name IN ({_PROJECTION_NAME_PLACEHOLDERS}) " "ORDER BY projection_name", - (workspace.workspace_id,), + (workspace.workspace_id, *_PROJECTION_NAME_PARAMETERS), ).fetchall() values: list[object] = [] for row in rows: @@ -780,7 +786,6 @@ def _read_projection_freshness_sync( root_hash = row["source_event_root_hash"] if ( not isinstance(projection, str) - or projection not in _PROJECTION_NAMES or not isinstance(root_hash, str) or _SHA256_RE.fullmatch(root_hash) is None ): @@ -802,13 +807,12 @@ def _read_projection_freshness_sync( "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " "THEN 1 ELSE 0 END) AS marked_stale " "FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({_PROJECTION_NAME_PLACEHOLDERS}) " "GROUP BY projection_name", - (workspace.workspace_id,), + (workspace.workspace_id, *_PROJECTION_NAME_PARAMETERS), ).fetchall() stale = 0 for row in state_rows: - if row["projection_name"] not in _PROJECTION_NAMES: - raise ValueError("projection name is invalid") if ( _plain_int(row["has_active"]) == 0 or _plain_int(row["marked_stale"]) == 1 @@ -1104,9 +1108,7 @@ def _record_summary( deleted = row["deleted_at_us"] is not None if deleted: _datetime_from_us(row["deleted_at_us"]) - relative_path = row["file_path_relative"] - if relative_path is not None and not isinstance(relative_path, str): - raise ValueError("relative file path is invalid") + relative_path = stored_relative_path(row["file_path_relative"]) status = "archived" if archived else "invalidated" if deleted else "current" public_value = { "record_id": row["record_id"], diff --git a/daem0nmcp/retrieval/lexical_config.py b/daem0nmcp/retrieval/lexical_config.py index d23c243..337f286 100644 --- a/daem0nmcp/retrieval/lexical_config.py +++ b/daem0nmcp/retrieval/lexical_config.py @@ -6,6 +6,13 @@ import json import re +# The projections a retrieval provider serves. ``projection_manifests`` also +# carries the migration's table snapshots (``memory_records`` and friends), so +# readers of retrieval freshness must select these names instead of refusing +# every other one. +RETRIEVAL_PROJECTION_NAMES = frozenset( + {"lexical", "dense", "graph", "temporal", "procedure", "outcome", "code"} +) LEXICAL_BM25_WEIGHTS = (1.0, 0.7, 1.5) LEXICAL_TOKENIZER = "unicode61 remove_diacritics 2" LEXICAL_BUILD_CONFIG = { @@ -48,6 +55,7 @@ def lexical_fts_table_name(workspace_id: str, generation: int) -> str: __all__ = [ + "RETRIEVAL_PROJECTION_NAMES", "LEXICAL_BM25_WEIGHTS", "LEXICAL_BUILD_CONFIG", "LEXICAL_TOKENIZER", diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index be1003b..2c36240 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -35,6 +35,7 @@ ) from .migrations import MIGRATIONS from .migrations.v7 import _source_row_hash, inventory_database +from .retrieval.lexical_config import RETRIEVAL_PROJECTION_NAMES from .schema_version import CURRENT_SCHEMA_VERSION, REQUIRED_V7_SCHEMA_VERSIONS from .storage_activation import ( ActiveDatabasePointer, @@ -74,6 +75,17 @@ "workspace_link_events", } ) +# The snapshots `migrate-v7` records for the tables it fills directly; no +# retrieval provider serves them. +_TABLE_SNAPSHOT_PROJECTIONS = frozenset( + { + "memory_records", + "memory_fact_versions", + "memory_relationship_versions", + "entities", + "communities", + } +) _LOCAL_PROJECTIONS = frozenset( { "memory_records", @@ -86,12 +98,7 @@ "outcome", } ) -_SUPPORTED_PROJECTIONS = _LOCAL_PROJECTIONS | { - "dense", - "code", - "entities", - "communities", -} +_SUPPORTED_PROJECTIONS = _TABLE_SNAPSHOT_PROJECTIONS | RETRIEVAL_PROJECTION_NAMES _GENERATION_TABLES = { "lexical": "retrieval_documents", "procedure": "record_procedures", @@ -709,6 +716,12 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: previous = latest_local.get(key) if previous is None or int(row[6]) > previous[0]: latest_local[key] = (int(row[6]), not current or declared_stale) + if projection_name in _TABLE_SNAPSHOT_PROJECTIONS: + # A migration snapshot records what `migrate-v7` built for a table + # the event store owns, not a rebuildable retrieval generation, so + # later writes move the event cursor and the row count past it by + # design. Only the retrieval projections carry those invariants. + continue if row[2] == "active" and not declared_stale: if projection_name == "code": if (row[8], row[9]) != (None, None): From 0d60ca7b5e4310a001e8cd442dc6ec20f0f89310 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:47:13 -0400 Subject: [PATCH 49/72] fix: read a migrated v6 record instead of calling it corrupt v6 stored a host-absolute `file_path` for every file-linked memory, and a relative form that is `../outside/x.py` for a file above the project (the absolute path itself across Windows drives). Every v7 reader treated a non-null `file_path` as corruption, so one migrated row made the whole recall, outcome or evidence hydration fail; the `../` form then failed the wire's relative-path rule on the way out. The readers now treat the legacy column as provenance: they never emit it, and `stored_relative_path` turns a stored relative path into one the wire accepts or into nothing at all. `relative_to_root` decides containment with `Path.relative_to` on resolved, normalized paths, so `C:\proj2\a.py` is not under `C:\proj`; it is also the one predicate behind `RelativePath`. `RecordSummary` is built only from storage, so it clips what v6 had no limit for -- tag length, tag uniqueness, tag count, empty content -- and shows the earlier of valid and transaction time as the creation instead of refusing the record. A v6 memory recorded with a backdated valid time was enough to fail `session_brief` for the whole workspace. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/intelligence_operations.py | 3 +- daem0nmcp/api/v7/local_state_operations.py | 3 +- daem0nmcp/api/v7/maintenance_operations.py | 11 ++- daem0nmcp/api/v7/models.py | 88 ++++++++++++++++----- daem0nmcp/api/v7/operations.py | 13 ++- daem0nmcp/api/v7/record_operations.py | 19 ++--- daem0nmcp/api/v7/runtime_services.py | 19 +++-- daem0nmcp/workspace.py | 53 +++++++++++++ tests/api_v7/test_runtime_services.py | 11 +-- 9 files changed, 172 insertions(+), 48 deletions(-) diff --git a/daem0nmcp/api/v7/intelligence_operations.py b/daem0nmcp/api/v7/intelligence_operations.py index c94d358..cdd4644 100644 --- a/daem0nmcp/api/v7/intelligence_operations.py +++ b/daem0nmcp/api/v7/intelligence_operations.py @@ -35,6 +35,7 @@ EvidenceRef, RecordSummary, parse_wire_datetime, + stored_relative_path, ) from .public_ids import PublicObjectIdNotFound, PublicObjectIdRepository from .resources import RuleView @@ -436,7 +437,7 @@ def _record_evidence_from_events( content = _safe_text(state.get("content")) try: tags = state.get("tags", []) - relative_path = state.get("file_path_relative") + relative_path = stored_relative_path(state.get("file_path_relative")) content_hash = memory_content_hash(state) recorded_at_us = int(row["recorded_at_us"]) created = _datetime_from_us(created_at_us) diff --git a/daem0nmcp/api/v7/local_state_operations.py b/daem0nmcp/api/v7/local_state_operations.py index c222eb7..c83969d 100644 --- a/daem0nmcp/api/v7/local_state_operations.py +++ b/daem0nmcp/api/v7/local_state_operations.py @@ -42,6 +42,7 @@ MutationReceipt, RecordSummary, parse_wire_datetime, + stored_relative_path, ) from .resources import ActiveContextItem from .runtime_services import WorkspaceStorageResolver @@ -305,7 +306,7 @@ def _record_summary(row: sqlite3.Row) -> RecordSummary: "record_type": record_type, "excerpt": content[:4000], "tags": _json_string_list(row["tags_json"]), - "relative_file_path": row["file_path_relative"], + "relative_file_path": stored_relative_path(row["file_path_relative"]), "current_status": status, "content_hash": row["content_hash"], "created_at": _datetime_from_us(row["created_at_us"]), diff --git a/daem0nmcp/api/v7/maintenance_operations.py b/daem0nmcp/api/v7/maintenance_operations.py index 2a07ee6..977bc98 100644 --- a/daem0nmcp/api/v7/maintenance_operations.py +++ b/daem0nmcp/api/v7/maintenance_operations.py @@ -35,7 +35,12 @@ from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import DestructiveMutationReceipt, Preview, RecordSummary +from .models import ( + DestructiveMutationReceipt, + Preview, + RecordSummary, + stored_relative_path, +) from .runtime_protocols import ActiveStorageResolver, WorkerPool from .runtime_services import WorkspaceStorageResolver from .tasks import await_task_terminal @@ -672,7 +677,9 @@ def _record_summary(record: _RecordSnapshot) -> RecordSummary: "record_type": record.state["record_type"], "excerpt": content[:4000], "tags": tags, - "relative_file_path": record.state["file_path_relative"], + "relative_file_path": stored_relative_path( + record.state["file_path_relative"] + ), "current_status": status, "content_hash": record.content_hash, "created_at": created_at, diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index d655b71..998bee1 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -8,7 +8,6 @@ from __future__ import annotations import json -import posixpath import re from collections.abc import Iterator, Mapping from datetime import datetime, timezone @@ -36,6 +35,7 @@ ) from typing_extensions import Self, TypeAliasType +from ...workspace import is_workspace_relative_path from .errors import INTERNAL_ERROR_MESSAGE, ErrorCode MAX_JSON_COLLECTION_ITEMS = 4096 @@ -149,20 +149,7 @@ def contains_absolute_filesystem_path(value: object) -> bool: def _relative_path(value: str) -> str: - if value == ".": - return value - if ( - not value - or "\\" in value - or "\x00" in value - or value.startswith(("/", "~")) - or _WINDOWS_DRIVE.match(value) is not None - ): - raise ValueError("path must be a normalized workspace-relative POSIX path") - components = value.split("/") - if any(component in {"", ".", ".."} for component in components): - raise ValueError("path must be a normalized workspace-relative POSIX path") - if posixpath.normpath(value) != value: + if not is_workspace_relative_path(value): raise ValueError("path must be a normalized workspace-relative POSIX path") return value @@ -655,7 +642,41 @@ class Page(WireModel, Generic[T]): truncated: bool +MIGRATED_EMPTY_CONTENT = "" + + +def stored_relative_path(value: object) -> str | None: + """Return a stored path only when it is a workspace-relative POSIX path. + + v6 stored an absolute ``file_path`` plus a relative form that is + ``../outside/x.py`` for a file above the project (and the absolute path + itself across Windows drives). Neither may leave the workspace, so a + migrated record reads back without its file link instead of failing. + """ + + if not isinstance(value, str): + return None + try: + return _relative_path(value) + except ValueError: + return None + + class RecordSummary(WireModel): + """The bounded public view of a stored record. + + It is only ever built from storage, never parsed from a caller, so it + clips rows that predate v7's bounds -- v6 tags had no length, + uniqueness or count limit, v6 content could be empty, and v6 kept + valid time and transaction time independently -- rather than denying + every read of a migrated workspace. A v7-written record already + satisfies the bounds, so clipping is a no-op for it. + + ``relative_file_path`` is a containment rule rather than a bound, so + it stays refused here; a stored value is normalized by + ``stored_relative_path`` where the row is read. + """ + record_id: RecordId record_type: RecordType excerpt: Annotated[ @@ -672,11 +693,37 @@ class RecordSummary(WireModel): created_at: AwareDateTime updated_at: AwareDateTime - @model_validator(mode="after") - def validate_timeline(self) -> RecordSummary: - if self.updated_at < self.created_at: - raise ValueError("updated_at cannot precede created_at") - return self + @model_validator(mode="before") + @classmethod + def clip_stored_record(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + values = dict(data) + tags = values.get("tags") + if isinstance(tags, (list, tuple)): + clipped: list[str] = [] + for tag in tags: + if not isinstance(tag, str) or not tag: + continue + tag = tag[:80] + if tag not in clipped: + clipped.append(tag) + values["tags"] = clipped[:32] + excerpt = values.get("excerpt") + if isinstance(excerpt, str): + values["excerpt"] = excerpt[:4000] or MIGRATED_EMPTY_CONTENT + # A record's creation carries its valid time, which may be backdated + # or in the future relative to the transaction time it was written at, + # and v6 kept the two independently. The summary shows the earlier of + # the two as the creation rather than refusing the record. + try: + created = parse_wire_datetime(values["created_at"]) + updated = parse_wire_datetime(values["updated_at"]) + except (KeyError, TypeError, ValueError): + return values + if updated < created: + values["created_at"] = updated + return values class EvidenceRef(WireModel): @@ -863,6 +910,7 @@ class Preview(WireModel): "FieldError", "guarded_strings", "is_host_absolute_path", + "stored_relative_path", "JsonObject", "JsonValue", "MAX_CONTEXT_JSON_BYTES", diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index ff1e6b4..d1c4179 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -53,7 +53,13 @@ from ...workspace import Workspace from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import EvidenceRef, Page, RecordSummary, RelativePath +from .models import ( + EvidenceRef, + Page, + RecordSummary, + RelativePath, + stored_relative_path, +) from .portable_projections import ( ImportFinalizationLease, PortableTransferError, @@ -244,6 +250,9 @@ def _active_connection( try: with DatabaseFileLock(storage, "shared"): active = resolve_active_database(storage) + if active.format_version == 6: + # An intact but un-migrated v6 store: retrying never helps. + raise CoreOperationError("MIGRATION_REQUIRED") if active.format_version != _FORMAT_VERSION: raise CoreOperationError("CAPABILITY_DEGRADED") connection = sqlite3.connect(active.path, timeout=5.0) @@ -633,7 +642,7 @@ def _summary( record = _record_from_event(event) content = record.get("content") tags = record.get("tags", []) - relative = record.get("file_path_relative") + relative = stored_relative_path(record.get("file_path_relative")) if not isinstance(content, str) or not content or not isinstance(tags, list): raise CoreOperationError("IMPORT_INVALID") if record.get("deleted_at_us") is not None: diff --git a/daem0nmcp/api/v7/record_operations.py b/daem0nmcp/api/v7/record_operations.py index c5d1276..1bdb3b4 100644 --- a/daem0nmcp/api/v7/record_operations.py +++ b/daem0nmcp/api/v7/record_operations.py @@ -51,6 +51,7 @@ Page, RecordSummary, parse_wire_datetime, + stored_relative_path, ) from .pinned import IdempotencyConflict from .runtime_services import WorkspaceStorageResolver @@ -307,8 +308,10 @@ def _load_record( ).fetchall() if not rows: raise RecordOperationError("NOT_FOUND") - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RecordOperationError("CAPABILITY_DEGRADED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it is + # legacy provenance, never emitted, so it must not deny the read. return rows[0] @@ -329,12 +332,10 @@ def _summary( if not isinstance(content, str) or not content: raise RecordOperationError("CAPABILITY_DEGRADED") tags = _parse_json(row["tags_json"], list) + # A future valid-time may be recorded now; ``RecordSummary`` exposes the + # transaction-time creation while the event retains the exact valid time. created_at = _datetime_from_us(row["created_at_us"]) updated_at = _datetime_from_us(row["updated_at_us"]) - # A future valid-time may be recorded now. The bounded summary exposes - # its transaction-time creation while the event retains exact valid-time. - if created_at > updated_at: - created_at = updated_at try: return RecordSummary.model_validate( { @@ -342,9 +343,9 @@ def _summary( "record_type": str(row["record_type"]), "excerpt": content[:4000], "tags": tags if include_metadata else [], - "relative_file_path": None - if not include_metadata or row["file_path_relative"] is None - else str(row["file_path_relative"]), + "relative_file_path": stored_relative_path(row["file_path_relative"]) + if include_metadata + else None, "current_status": _record_status(row), "content_hash": str(row["content_hash"]), "created_at": created_at, @@ -366,7 +367,7 @@ def _record_state(row: sqlite3.Row) -> dict[str, Any]: "context": _parse_json(row["context_json"], dict), "tags": _parse_json(row["tags_json"], list), "file_path": None, - "file_path_relative": row["file_path_relative"], + "file_path_relative": stored_relative_path(row["file_path_relative"]), "keywords": row["keywords"], "is_permanent": bool(row["is_permanent"]), "pinned": bool(row["pinned"]), diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index 09d08c3..6b40a0b 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -63,6 +63,7 @@ RecordSummary, RetrievalData, TokenUsage, + stored_relative_path, ) from .models import ( EvidenceItem as PublicEvidenceItem, @@ -187,6 +188,9 @@ def locked_active( workspace: Workspace, ) -> Iterator[ResolvedActiveDatabase]: with self.locked_current(workspace) as active: + if active.format_version == 6: + # An intact but un-migrated v6 store: retrying never helps. + raise RuntimeServiceError("MIGRATION_REQUIRED") if ( active.pointer is None or active.format_version != _FORMAT_VERSION @@ -441,8 +445,9 @@ def _load_record_row( ).fetchall() if len(rows) != 1: raise RuntimeServiceError("NOT_FOUND") - if rows[0]["file_path"] is not None: - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote. It is + # never emitted (``_record_summary`` and ``_record_state`` drop it), so it + # is legacy provenance, not corruption. return rows[0] @@ -462,9 +467,7 @@ def _record_summary( "record_type": str(row["record_type"]), "excerpt": content[:4000], "tags": tags, - "relative_file_path": None - if row["file_path_relative"] is None - else str(row["file_path_relative"]), + "relative_file_path": stored_relative_path(row["file_path_relative"]), "current_status": _record_status(row, evidence_status), "content_hash": str(row["content_hash"]), "created_at": _datetime_from_us(row["created_at_us"]), @@ -488,7 +491,7 @@ def _record_state(row: sqlite3.Row) -> dict[str, Any]: "context": context, "tags": tags, "file_path": None, - "file_path_relative": row["file_path_relative"], + "file_path_relative": stored_relative_path(row["file_path_relative"]), "keywords": row["keywords"], "is_permanent": bool(row["is_permanent"]), "pinned": bool(row["pinned"]), @@ -1303,7 +1306,7 @@ def _hydrate_candidates( "AND event.stream_id=record.record_id LIMIT 2", (query.workspace_id, *key), ).fetchall() - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") hydrated[key] = rows[0] @@ -1435,7 +1438,7 @@ def _hydrate( "AND event.stream_id=record.record_id LIMIT 2", (query.workspace_id, *key), ).fetchall() - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") hydrated[key] = rows[0] diff --git a/daem0nmcp/workspace.py b/daem0nmcp/workspace.py index f3ce449..e598f02 100644 --- a/daem0nmcp/workspace.py +++ b/daem0nmcp/workspace.py @@ -5,10 +5,14 @@ import hashlib import json import os +import posixpath +import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path +_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") + class WorkspaceAccessError(ValueError): """Raised when a caller selects a workspace that is not registered.""" @@ -182,6 +186,55 @@ def resolve(self, selector: str | os.PathLike[str] | None) -> Workspace: return workspace +def is_workspace_relative_path(value: object) -> bool: + """Return whether a string is a normalized workspace-relative POSIX path.""" + + if not isinstance(value, str) or not 1 <= len(value) <= 1024: + return False + if value == ".": + return True + if ( + "\\" in value + or "\x00" in value + or value.startswith(("/", "~")) + or _WINDOWS_DRIVE.match(value) is not None + ): + return False + if any(component in {"", ".", ".."} for component in value.split("/")): + return False + return posixpath.normpath(value) == value + + +def relative_to_root( + path: str | os.PathLike[str], + root: str | os.PathLike[str], +) -> str | None: + """Return ``path`` inside ``root`` as a POSIX relative path, else ``None``. + + Containment is decided with ``Path.relative_to`` on resolved, normalized + paths, never with a string prefix, so ``C:\\proj2\\a.py`` is not treated as + living under ``C:\\proj``. The returned string always satisfies the wire's + relative-path rule. + """ + + try: + resolved_root = normalize_resolved_path(Path(root).expanduser().resolve()) + resolved = normalize_resolved_path(Path(path).expanduser().resolve()) + depth = len( + Path(os.path.normcase(str(resolved))) + .relative_to(os.path.normcase(str(resolved_root))) + .parts + ) + except (OSError, RuntimeError, ValueError): + return None + if depth < 1: + return None + # Join the resolved path's own tail so the result keeps the filesystem's + # spelling rather than the case-folded form containment was decided on. + value = "/".join(resolved.parts[-depth:]) + return value if is_workspace_relative_path(value) else None + + def _has_parent_component(value: str) -> bool: normalized = value.replace("\\", "/") return ".." in normalized.split("/") diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index ffffc8d..afb81b1 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -125,7 +125,7 @@ async def test_safe_resolver_requires_a_format_7_active_pointer(self) -> None: (other_storage / "daem0nmcp.db").touch() pointerless = WorkspaceRegistry([other], default_root=other).default with ( - self.assertRaisesRegex(RuntimeServiceError, "ACTIVE_V7_UNAVAILABLE"), + self.assertRaisesRegex(RuntimeServiceError, "MIGRATION_REQUIRED"), resolver.locked_active(pointerless), ): self.fail("pointerless storage was admitted") @@ -830,16 +830,17 @@ async def retrieve(self, _query): ): await run(self._retrieval_result(stored, provider="")) + # A row migrated from v6 keeps the host-absolute ``file_path`` v6 + # wrote. It is legacy provenance, so it must not deny the recall, and + # it must never leave the server. with closing(sqlite3.connect(self.database)) as connection: connection.execute( "UPDATE memory_records SET file_path=? WHERE record_id=?", (str(self.root / "private.txt"), stored.record.record_id), ) connection.commit() - with self.assertRaisesRegex( - RuntimeServiceError, "EVIDENCE_AUTHENTICATION_FAILED" - ): - await run(self._retrieval_result(stored)) + legacy = await run(self._retrieval_result(stored)) + self.assertNotIn("private.txt", legacy.model_dump_json()) async def test_recall_rejects_federation_and_preserves_abstention( self, From fc5f57fd0a935d5b7be40d62e5ab69874f63fbaa Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:47:46 -0400 Subject: [PATCH 50/72] fix: migrate v6 rows into values v7 can read, and say what changed The migration copied each v6 memory verbatim, so the record it wrote could not be read back: the host-absolute `file_path`, a `../outside` relative path, a 120-character tag, duplicate tags, 43 tags and empty content all pass in v6 and all fail the v7 wire. `_memory_state` now writes `file_path=NULL` and a workspace-relative link derived from the stored relative form, or re-derived from the absolute path against the workspace root -- or against the single project root the v6 database records, which is this workspace's own history after the project directory was moved. Tags are clipped to the wire's bounds and empty content becomes the same marker the reader uses. Every raw value stays in the event's lossless `legacy` payload, which keeps `source_row_hash` provenance exact. `validation` now reports rule and trigger counts, dropped file links, foreign triggers and reconciled governance, and `MigrationResult` carries them as warnings instead of reporting a silent success. A trigger is this workspace's when the database records exactly one project path; the rest are counted, not dropped silently. Governance is reconciled against the v6 tables before the candidate is ready, so a rule edited (or deleted) after a legacy command already backfilled migration 21 in place no longer keeps its stale `must_not`. `rollback` refuses with ROLLBACK_WOULD_HIDE_WRITES, naming the count and the retained candidate, when events were recorded since activation; `--discard-v7-writes` proceeds and says what stayed behind. The run row also records the published `candidate.db`, not the partial name it was built under, so `verify-v7` no longer calls a just-activated migration invalid. Co-Authored-By: Claude Opus 5 --- daem0nmcp/migrations/schema.py | 405 ++++++++++++++++++++++----------- daem0nmcp/migrations/v7.py | 245 ++++++++++++++++++-- 2 files changed, 500 insertions(+), 150 deletions(-) diff --git a/daem0nmcp/migrations/schema.py b/daem0nmcp/migrations/schema.py index 8371fb5..5062e6f 100644 --- a/daem0nmcp/migrations/schema.py +++ b/daem0nmcp/migrations/schema.py @@ -12,6 +12,7 @@ import time from datetime import datetime, timezone from pathlib import Path +from typing import Any from daem0nmcp.schema_version import CURRENT_SCHEMA_VERSION @@ -2821,6 +2822,136 @@ def _retained_path_workspace_id(value: object) -> str | None: return f"ws_{hashlib.sha256(root_key.encode('utf-8')).hexdigest()[:24]}" +def _retained_source_id(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + return value + + +def _retained_rule_states( + connection: sqlite3.Connection, + workspace_id: str, +) -> dict[str, tuple[int, dict[str, Any]]]: + """The canonical rule state each retained v6 rule row stands for.""" + + if not _table_exists(connection, "rules"): + return {} + states: dict[str, tuple[int, dict[str, Any]]] = {} + for row in connection.execute( + "SELECT id,trigger,must_do,must_not,ask_first,warnings,priority," + "enabled,created_at FROM rules ORDER BY id" + ).fetchall(): + source_id = _retained_source_id(row[0]) + if not isinstance(row[1], str) or not 1 <= len(row[1]) <= 2_000: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + if ( + isinstance(row[6], bool) + or not isinstance(row[6], int) + or not -1_000 <= row[6] <= 1_000 + or row[7] not in (0, 1) + ): + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + public_id = _retained_public_mapping( + connection, workspace_id, "rule", source_id + ) + created_at_us = _retained_timestamp_us(row[8]) + states[public_id] = ( + source_id, + { + "rule_id": public_id, + "trigger": row[1], + "must_do": _retained_text_list(row[2]), + "must_not": _retained_text_list(row[3]), + "ask_first": _retained_text_list(row[4]), + "warnings": _retained_text_list(row[5]), + "priority": row[6], + "enabled": bool(row[7]), + "created_at_us": created_at_us, + "updated_at_us": created_at_us, + }, + ) + return states + + +def _retained_trigger_states( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[dict[str, tuple[int, dict[str, Any]]], int]: + """The canonical trigger states, plus the count left to another project. + + v6 recorded the project root each trigger belongs to. When the database + records exactly one, it is this workspace's own history even if the project + directory has since been moved, so its triggers migrate. With several + roots, only the ones naming this workspace do. + """ + + if not _table_exists(connection, "context_triggers"): + return {}, 0 + rows = connection.execute( + "SELECT id,project_path,trigger_type,pattern,recall_topic,recall_categories," + "is_active,priority,created_at FROM context_triggers ORDER BY id" + ).fetchall() + recorded = {row[1] for row in rows if isinstance(row[1], str) and row[1]} + sole_root = recorded.pop() if len(recorded) == 1 else None + public_types = { + "file_pattern": "file", + "tag_match": "tag", + "entity_match": "entity", + } + states: dict[str, tuple[int, dict[str, Any]]] = {} + foreign = 0 + for row in rows: + source_id = _retained_source_id(row[0]) + if row[1] != sole_root and _retained_path_workspace_id(row[1]) != workspace_id: + foreign += 1 + continue + trigger_type = public_types.get(row[2]) + if trigger_type is None: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + if ( + not isinstance(row[3], str) + or not 1 <= len(row[3]) <= 2_000 + or not isinstance(row[4], str) + or not 1 <= len(row[4]) <= 2_000 + or row[6] not in (0, 1) + or isinstance(row[7], bool) + or not isinstance(row[7], int) + ): + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + public_id = _retained_public_mapping( + connection, workspace_id, "trigger", source_id + ) + created_at_us = _retained_timestamp_us(row[8]) + states[public_id] = ( + source_id, + { + "trigger_id": public_id, + "trigger_type": trigger_type, + "pattern": row[3], + "recall_query": row[4], + "categories": _retained_text_list(row[5]), + "enabled": bool(row[6]), + "priority": row[7], + "created_at_us": created_at_us, + "updated_at_us": created_at_us, + "deleted_at_us": None, + }, + ) + return states, foreign + + +def _governance_stream_exists( + connection: sqlite3.Connection, workspace_id: str, stream_id: str +) -> bool: + return ( + connection.execute( + "SELECT 1 FROM governance_events WHERE workspace_id=? AND stream_id=?", + (workspace_id, stream_id), + ).fetchone() + is not None + ) + + def backfill_retained_governance( connection: sqlite3.Connection, workspace_id: str, @@ -2840,159 +2971,169 @@ def backfill_retained_governance( raise RuntimeError("GOVERNANCE_SCHEMA_INCOMPLETE") store = GovernanceEventStore(connection, assume_transaction=True) inserted = 0 - if _table_exists(connection, "rules"): - rows = connection.execute( - "SELECT id,trigger,must_do,must_not,ask_first,warnings,priority," - "enabled,created_at FROM rules ORDER BY id" - ).fetchall() - for row in rows: - source_id = row[0] - if ( - isinstance(source_id, bool) - or not isinstance(source_id, int) - or source_id < 1 - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - public_id = _retained_public_mapping( - connection, workspace_id, "rule", source_id - ) + trigger_states, _foreign = _retained_trigger_states(connection, workspace_id) + for kind, projection, column, event_type, states in ( + ( + "rule", + "governance_rules", + "rule_id", + "rule.created", + _retained_rule_states(connection, workspace_id), + ), + ( + "trigger", + "governance_context_triggers", + "trigger_id", + "context_trigger.created", + trigger_states, + ), + ): + for public_id, (source_id, state) in states.items(): if ( connection.execute( - "SELECT 1 FROM governance_rules WHERE workspace_id=? AND rule_id=?", + f"SELECT 1 FROM {projection} WHERE workspace_id=? AND {column}=?", (workspace_id, public_id), ).fetchone() is not None ): continue - if ( - connection.execute( - "SELECT 1 FROM governance_events WHERE workspace_id=? " - "AND stream_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): + if _governance_stream_exists(connection, workspace_id, public_id): raise RuntimeError("GOVERNANCE_BACKFILL_INTEGRITY_ERROR") - created_at_us = _retained_timestamp_us(row[8]) - if not isinstance(row[1], str) or not 1 <= len(row[1]) <= 2_000: - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if ( - isinstance(row[6], bool) - or not isinstance(row[6], int) - or not -1_000 <= row[6] <= 1_000 - or row[7] not in (0, 1) - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - state = { - "rule_id": public_id, - "trigger": row[1], - "must_do": _retained_text_list(row[2]), - "must_not": _retained_text_list(row[3]), - "ask_first": _retained_text_list(row[4]), - "warnings": _retained_text_list(row[5]), - "priority": row[6], - "enabled": bool(row[7]), - "created_at_us": created_at_us, - "updated_at_us": created_at_us, - } store.append_and_project( GovernanceEventCommand( workspace_id=workspace_id, stream_id=public_id, - stream_kind="rule", - event_type="rule.created", - occurred_at_us=created_at_us, - recorded_at_us=created_at_us, + stream_kind=kind, + event_type=event_type, + occurred_at_us=state["created_at_us"], + recorded_at_us=state["created_at_us"], actor_type="migration", - correlation_id=f"migration21:rule:{source_id}", + correlation_id=f"migration21:{kind}:{source_id}", payload=state, expected_stream_version=1, ) ) inserted += 1 - if _table_exists(connection, "context_triggers"): - rows = connection.execute( - "SELECT id,project_path,trigger_type,pattern,recall_topic,recall_categories," - "is_active,priority,created_at FROM context_triggers ORDER BY id" - ).fetchall() - public_types = { - "file_pattern": "file", - "tag_match": "tag", - "entity_match": "entity", + return inserted + + +def reconcile_retained_governance( + connection: sqlite3.Connection, + workspace_id: str, + *, + now_us: int, +) -> dict[str, int]: + """Carry v6 rule and trigger edits made after an earlier backfill into v7. + + A legacy command can apply migration 21 to a v6 store, and the v6 tables + may then be edited again before `migrate-v7` runs. The backfill skips a + stream that already exists, so without this pass the edit or the deletion + would never reach v7 and a `must_not` could silently disappear. + + The governance event vocabulary has `rule.updated` and + `context_trigger.deleted`; a rule that no longer exists in v6 is therefore + recorded as disabled rather than removed. + """ + + from daem0nmcp.event_store import ( + GovernanceEventCommand, + GovernanceEventStore, + ) + + if not _table_exists(connection, "governance_events"): + raise RuntimeError("GOVERNANCE_SCHEMA_INCOMPLETE") + store = GovernanceEventStore(connection, assume_transaction=True) + counts = {"rules_updated": 0, "triggers_deleted": 0} + + rule_states = _retained_rule_states(connection, workspace_id) + for row in connection.execute( + "SELECT rule_id,trigger,must_do_json,must_not_json,ask_first_json," + "warnings_json,priority,enabled,stream_version,created_at_us " + "FROM governance_rules WHERE workspace_id=? ORDER BY rule_id", + (workspace_id,), + ).fetchall(): + public_id = str(row[0]) + entry = rule_states.get(public_id) + current = { + "rule_id": public_id, + "trigger": row[1], + "must_do": json.loads(row[2]), + "must_not": json.loads(row[3]), + "ask_first": json.loads(row[4]), + "warnings": json.loads(row[5]), + "priority": row[6], + "enabled": bool(row[7]), } - for row in rows: - source_id = row[0] - if ( - isinstance(source_id, bool) - or not isinstance(source_id, int) - or source_id < 1 - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if _retained_path_workspace_id(row[1]) != workspace_id: - continue - public_id = _retained_public_mapping( - connection, workspace_id, "trigger", source_id - ) - if ( - connection.execute( - "SELECT 1 FROM governance_context_triggers " - "WHERE workspace_id=? AND trigger_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): - continue - if ( - connection.execute( - "SELECT 1 FROM governance_events WHERE workspace_id=? " - "AND stream_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INTEGRITY_ERROR") - trigger_type = public_types.get(row[2]) - if trigger_type is None: - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if ( - not isinstance(row[3], str) - or not 1 <= len(row[3]) <= 2_000 - or not isinstance(row[4], str) - or not 1 <= len(row[4]) <= 2_000 - or row[6] not in (0, 1) - or isinstance(row[7], bool) - or not isinstance(row[7], int) - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - created_at_us = _retained_timestamp_us(row[8]) - state = { - "trigger_id": public_id, - "trigger_type": trigger_type, - "pattern": row[3], - "recall_query": row[4], - "categories": _retained_text_list(row[5]), - "enabled": bool(row[6]), - "priority": row[7], - "created_at_us": created_at_us, - "updated_at_us": created_at_us, - "deleted_at_us": None, + if entry is None: + desired = dict(current, enabled=False) + correlation = f"migration21:rule-removed:{public_id}" + else: + desired = { + key: value + for key, value in entry[1].items() + if key not in {"created_at_us", "updated_at_us"} } - store.append_and_project( - GovernanceEventCommand( - workspace_id=workspace_id, - stream_id=public_id, - stream_kind="trigger", - event_type="context_trigger.created", - occurred_at_us=created_at_us, - recorded_at_us=created_at_us, - actor_type="migration", - correlation_id=f"migration21:trigger:{source_id}", - payload=state, - expected_stream_version=1, - ) + correlation = f"migration21:rule-updated:{entry[0]}" + if desired == current: + continue + store.append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id=public_id, + stream_kind="rule", + event_type="rule.updated", + occurred_at_us=now_us, + recorded_at_us=now_us, + actor_type="migration", + correlation_id=correlation, + payload={ + **desired, + "created_at_us": int(row[9]), + "updated_at_us": now_us, + }, + expected_stream_version=int(row[8]) + 1, ) - inserted += 1 - return inserted + ) + counts["rules_updated"] += 1 + + trigger_states, _foreign = _retained_trigger_states(connection, workspace_id) + for row in connection.execute( + "SELECT trigger_id,trigger_type,pattern,recall_query,categories_json," + "enabled,priority,stream_version,created_at_us FROM " + "governance_context_triggers WHERE workspace_id=? AND deleted_at_us IS NULL " + "ORDER BY trigger_id", + (workspace_id,), + ).fetchall(): + public_id = str(row[0]) + if public_id in trigger_states: + continue + store.append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id=public_id, + stream_kind="trigger", + event_type="context_trigger.deleted", + occurred_at_us=now_us, + recorded_at_us=now_us, + actor_type="migration", + correlation_id=f"migration21:trigger-removed:{public_id}", + payload={ + "trigger_id": public_id, + "trigger_type": row[1], + "pattern": row[2], + "recall_query": row[3], + "categories": json.loads(row[4]), + "enabled": bool(row[5]), + "priority": row[6], + "created_at_us": int(row[8]), + "updated_at_us": now_us, + "deleted_at_us": now_us, + }, + expected_stream_version=int(row[7]) + 1, + ) + ) + counts["triggers_deleted"] += 1 + return counts def _has_retained_public_rows(connection: sqlite3.Connection) -> bool: diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index ac55d19..38bc211 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -43,10 +43,26 @@ resolve_active_database, write_active_pointer, ) -from ..workspace import WorkspaceRegistry, resolve_derived_path +from ..workspace import ( + WorkspaceRegistry, + is_workspace_relative_path, + relative_to_root, + resolve_derived_path, +) +from .schema import ( + _retained_trigger_states, + reconcile_retained_governance, +) TARGET_FORMAT_VERSION = 7 DEFAULT_BATCH_SIZE = 500 +# v6 accepted empty content; the v7 bounded summary needs at least one +# character. ``api/v7/models.MIGRATED_EMPTY_CONTENT`` uses the same marker for +# rows migrated before this normalization existed. +_EMPTY_CONTENT = "" +# The v7 wire limits on a record's tags, which v6 did not have. +_MAX_TAGS = 32 +_MAX_TAG_CHARS = 80 _KNOWN_MEMORY_TYPES = { "decision", "pattern", @@ -715,8 +731,80 @@ def _legacy_score(value: Any) -> float | None: return number if math.isfinite(number) and 0 <= number <= 1 else None +def _legacy_project_roots( + connection: sqlite3.Connection, workspace_root: Path +) -> tuple[str, ...]: + """The roots a v6 file link may be relative to, workspace root first. + + A v6 database records the project root it was written for. When it holds + exactly one and it is not this workspace, the project was moved before the + migration, so file links are resolved against it as well. + """ + + recorded: set[str] = set() + for table in _table_names(connection): + if table in _V7_TABLE_NAMES: + continue + columns = { + str(row[1]) + for row in connection.execute( + f"PRAGMA table_info({_quoted_identifier(table)})" + ) + } + if "project_path" not in columns: + continue + recorded.update( + value + for value in _distinct(connection, table, "project_path") + if isinstance(value, str) and value + ) + roots = [str(workspace_root)] + if len(recorded) == 1: + recorded_root = next(iter(recorded)) + if os.path.normcase(recorded_root) != os.path.normcase(roots[0]): + roots.append(recorded_root) + return tuple(roots) + + +def _legacy_relative_path(row: sqlite3.Row, roots: Iterable[str]) -> str | None: + """Resolve a v6 file link to a workspace-relative path, or drop it. + + v6 stored a host-absolute ``file_path`` plus a relative form that is + ``../outside/x.py`` for a file above the project root (and the absolute + path itself across Windows drives). Neither may be emitted, so the stored + relative form is kept when it already is workspace-relative and otherwise + re-derived from the absolute path. Both raw values stay in the event's + lossless ``legacy`` payload either way. + """ + + stored = _column(row, "file_path_relative") + if is_workspace_relative_path(stored): + return str(stored) + absolute = _column(row, "file_path") + if not isinstance(absolute, str) or not absolute: + return None + for root in roots: + relative = relative_to_root(absolute, root) + if relative is not None: + return relative + return None + + +def _legacy_tags(values: Iterable[Any]) -> list[str]: + """Clip v6 tags to the v7 wire limits; the raw list stays in ``legacy``.""" + + clipped: list[str] = [] + for value in values: + if not isinstance(value, str) or not value: + continue + tag = value[:_MAX_TAG_CHARS] + if tag not in clipped: + clipped.append(tag) + return clipped[:_MAX_TAGS] + + def _memory_state( - row: sqlite3.Row, *, base: dict[str, Any] | None = None + row: sqlite3.Row, *, roots: Iterable[str] = (), base: dict[str, Any] | None = None ) -> dict[str, Any]: category = _column(row, "category") if base is None else base["record_type"] legacy_type = None @@ -727,7 +815,7 @@ def _memory_state( else: legacy_type = base["legacy_type"] context, _ = _parse_legacy_json(_column(row, "context"), dict, {}) - tags, _ = _parse_legacy_json(_column(row, "tags"), list, []) + raw_tags, _ = _parse_legacy_json(_column(row, "tags"), list, []) worked_value = _column(row, "worked") worked = None if worked_value is None else _legacy_bool(worked_value) content = _column(row, "content", "") @@ -736,16 +824,16 @@ def _memory_state( return { "record_type": category, "legacy_type": legacy_type, - "content": content, + "content": content or _EMPTY_CONTENT, "rationale": _column(row, "rationale"), "context": context, - "tags": tags, - "file_path": base.get("file_path") - if base is not None - else _column(row, "file_path"), + "tags": _legacy_tags(raw_tags), + # The host-absolute v6 path never reaches the v7 record; it survives + # in the event's lossless ``legacy`` payload. + "file_path": None, "file_path_relative": base.get("file_path_relative") if base is not None - else _column(row, "file_path_relative"), + else _legacy_relative_path(row, roots), "keywords": base.get("keywords") if base is not None else _column(row, "keywords"), @@ -838,6 +926,7 @@ def _projected_memory_state(row: sqlite3.Row) -> dict[str, Any]: def _validate_retained_compatibility( connection: sqlite3.Connection, workspace_id: str, + roots: tuple[str, ...] = (), ) -> None: """Compare current retained-v6 rows to their unique live typed claims.""" @@ -875,7 +964,7 @@ def _validate_retained_compatibility( (workspace_id, stream_id), ).fetchone() if projected is None or canonical_json_bytes( - _memory_state(source) + _memory_state(source, roots=roots) ) != canonical_json_bytes(_projected_memory_state(projected)): raise MigrationV7Error( "VALIDATION_FAILED", "current memory compatibility row diverges" @@ -1013,6 +1102,37 @@ def relationship_endpoint(value: Any) -> str: ) +def _migration_warnings(validation: dict[str, Any]) -> tuple[str, ...]: + """State every migrated fact that is not a faithful copy of the v6 row.""" + + warnings: list[str] = [] + dropped = int(validation.get("dropped_file_link_count", 0)) + if dropped: + warnings.append( + f"{dropped} memory file link(s) pointed outside the workspace and were " + "kept only in the legacy event payload" + ) + foreign = int(validation.get("foreign_trigger_count", 0)) + if foreign: + warnings.append( + f"{foreign} context trigger(s) belong to another project path and were " + "not migrated" + ) + updated = int(validation.get("rules_updated", 0)) + if updated: + warnings.append( + f"{updated} rule(s) changed in v6 after an earlier in-place upgrade and " + "were reconciled" + ) + deleted = int(validation.get("triggers_deleted", 0)) + if deleted: + warnings.append( + f"{deleted} context trigger(s) removed in v6 after an earlier in-place " + "upgrade were marked deleted" + ) + return tuple(warnings) + + def _run_migration_16(path: Path, workspace_id: str) -> None: from .schema import run_migrations @@ -1290,7 +1410,7 @@ def _mark_candidate_rolled_back(self, candidate: Path, run_id: str) -> None: connection.close() def _recover_ready_pointer( - self, resolved, workspace_id: str + self, resolved, workspace_id: str, workspace_root: Path ) -> MigrationResult | None: """Finish activation when the pointer was durable before metadata commit.""" @@ -1313,6 +1433,7 @@ def _recover_ready_pointer( run_id, workspace_id, inventory, + workspace_root, retained_authority=prior_validation.get("authority") == "retained_v7", ) self._bootstrap_lexical(resolved.path, workspace_id) @@ -1414,7 +1535,7 @@ def apply( if upgrade is not None: return upgrade recovered = self._recover_ready_pointer( - resolved, workspace.workspace_id + resolved, workspace.workspace_id, workspace.root ) if recovered is not None: return recovered @@ -1451,12 +1572,14 @@ def apply( run_id, workspace.workspace_id, batch_size, + workspace.root, ) validation = self._finalize_candidate( candidate, run_id, workspace.workspace_id, inventory, + workspace.root, retained_authority=action == "reactivate", ) self._bootstrap_lexical(candidate, workspace.workspace_id) @@ -1521,6 +1644,7 @@ def apply( inventory=inventory, checkpoints=checkpoints, validation=validation, + warnings=_migration_warnings(validation), ) except MigrationInterrupted: raise @@ -1633,7 +1757,8 @@ def _initialize_candidate( snapshot_hash, int(inventory["max_schema_version"]), "source.snapshot.db", - "candidate.db.partial", + # The final published artifact, as verify-v7 expects. + "candidate.db", canonical_json_bytes(inventory).decode("utf-8"), now, now, @@ -1890,12 +2015,14 @@ def _import_candidate( run_id: str, workspace_id: str, batch_size: int, + workspace_root: Path, ) -> dict[str, Any]: connection = sqlite3.connect(candidate) connection.row_factory = sqlite3.Row try: connection.execute("PRAGMA foreign_keys=ON") connection.execute("PRAGMA synchronous=FULL") + roots = _legacy_project_roots(connection, workspace_root) for table, importer in ( ("memories", self._import_memory_row), ("facts", self._import_fact_row), @@ -1934,7 +2061,9 @@ def _import_candidate( break store = EventStore(connection) for row in rows: - importer(connection, store, row, run_id, workspace_id) + importer( + connection, store, row, run_id, workspace_id, roots + ) row_hash = _source_row_hash(row, table) rolling = sha256_json([rolling, row_hash]) last_pk = int(row["id"]) @@ -1976,12 +2105,13 @@ def _import_memory_row( row: sqlite3.Row, run_id: str, workspace_id: str, + roots: tuple[str, ...], ) -> None: legacy_id = str(row["id"]) record_id = deterministic_id( "mem", "memory", workspace_id, "legacy", "memories", legacy_id ) - state = _memory_state(row) + state = _memory_state(row, roots=roots) stream_version = 0 versions = connection.execute( "SELECT * FROM memory_versions WHERE memory_id=? ORDER BY version_number, id", @@ -1996,7 +2126,7 @@ def _import_memory_row( } for version in versions: stream_version += 1 - version_state = _memory_state(version, base=state) + version_state = _memory_state(version, roots=roots, base=state) occurred, valid_quality, valid_original = _parse_legacy_time( _column(version, "valid_from") or _column(version, "changed_at") ) @@ -2167,6 +2297,7 @@ def _import_fact_row( row: sqlite3.Row, run_id: str, workspace_id: str, + roots: tuple[str, ...], ) -> None: legacy_id = str(row["id"]) fact_id = deterministic_id( @@ -2252,6 +2383,7 @@ def _import_relationship_row( row: sqlite3.Row, run_id: str, workspace_id: str, + roots: tuple[str, ...], ) -> None: occurred, quality, original = _parse_legacy_time(_column(row, "created_at")) source = self._mapped_memory( @@ -2335,6 +2467,7 @@ def _finalize_candidate( run_id: str, workspace_id: str, source_inventory: dict[str, Any], + workspace_root: Path, *, retained_authority: bool = False, ) -> dict[str, Any]: @@ -2342,12 +2475,22 @@ def _finalize_candidate( connection.row_factory = sqlite3.Row try: connection.execute("PRAGMA foreign_keys=ON") + roots = _legacy_project_roots(connection, workspace_root) connection.execute("BEGIN IMMEDIATE") now = self._clock_us() connection.execute( "UPDATE v7_migration_runs SET status='validating', updated_at_us=? WHERE migration_run_id=?", (now, run_id), ) + # A legacy command may already have backfilled governance into the + # v6 file; the v6 tables can have changed since. Carry those edits + # over before the candidate is declared ready. + governance = reconcile_retained_governance( + connection, workspace_id, now_us=now + ) + _foreign_trigger_states, foreign_triggers = _retained_trigger_states( + connection, workspace_id + ) quick = [ str(row[0]) for row in connection.execute("PRAGMA integrity_check") ] @@ -2471,7 +2614,7 @@ def _finalize_candidate( if table == "memories" and ( imported[2] != "legacy.memory_state_imported" or canonical_json_bytes(imported_payload.get("record")) - != canonical_json_bytes(_memory_state(source_row)) + != canonical_json_bytes(_memory_state(source_row, roots=roots)) ): raise MigrationV7Error( "VALIDATION_FAILED", @@ -2864,7 +3007,7 @@ def _finalize_candidate( ) if retained_authority: - _validate_retained_compatibility(connection, workspace_id) + _validate_retained_compatibility(connection, workspace_id, roots) expected_maps = { "memories": int(source_inventory["memory_count"]), @@ -2949,6 +3092,16 @@ def _finalize_candidate( now, ), ) + dropped_file_links = 0 + if _table_exists(connection, "memories"): + for source_row in connection.execute("SELECT * FROM memories"): + absolute = _column(source_row, "file_path") + if ( + isinstance(absolute, str) + and absolute + and _legacy_relative_path(source_row, roots) is None + ): + dropped_file_links += 1 validation = { "integrity_check": "ok", "foreign_key_violations": [], @@ -2959,6 +3112,12 @@ def _finalize_candidate( "relationship_version_count": _count( connection, "memory_relationship_versions" ), + "rule_count": _count(connection, "governance_rules"), + "trigger_count": _count(connection, "governance_context_triggers"), + "rules_updated": governance["rules_updated"], + "triggers_deleted": governance["triggers_deleted"], + "foreign_trigger_count": foreign_triggers, + "dropped_file_link_count": dropped_file_links, } if retained_authority: validation["authority"] = "retained_v7" @@ -3005,10 +3164,44 @@ def _restore_pointer(self, storage: Path, previous) -> None: return write_active_pointer(storage, previous.pointer) + def _events_after_activation(self, candidate: Path, run_id: str) -> int: + """Count canonical events written to the candidate since activation. + + Rolling the pointer back to v6 makes them unreachable: the v6 file + never received them, and a later v6 write changes the run identity, so + the next `--apply` builds a different candidate. + """ + + connection = sqlite3.connect(candidate) + try: + row = connection.execute( + "SELECT activated_at_us FROM v7_migration_runs WHERE migration_run_id=?", + (run_id,), + ).fetchone() + if row is None or row[0] is None: + return 0 + activated = int(row[0]) + total = 0 + for table in ("memory_events", "governance_events"): + if not _table_exists(connection, table): + continue + total += int( + connection.execute( + f"SELECT count(*) FROM {_quoted_identifier(table)} " + "WHERE recorded_at_us>?", + (activated,), + ).fetchone()[0] + ) + return total + finally: + connection.close() + def rollback( self, selector: str | os.PathLike[str] | None = None, migration_run_id: str | None = "latest", + *, + discard_v7_writes: bool = False, ) -> MigrationResult: workspace, storage = self._storage(selector) with DatabaseFileLock(storage, "exclusive"): @@ -3099,6 +3292,14 @@ def rollback( workspace.workspace_id, error_code="ROLLBACK_STATE_INVALID", ) + hidden = self._events_after_activation(resolved.path, run_id) + if hidden and not discard_v7_writes: + raise MigrationV7Error( + "ROLLBACK_WOULD_HIDE_WRITES", + f"{hidden} event(s) recorded since activation would become " + f"unreachable; they stay in {current_relative}. Re-run with " + "--discard-v7-writes to roll back anyway", + ) pointer = ActiveDatabasePointer( format_version=source_format, generation=resolved.generation + 1, @@ -3116,6 +3317,14 @@ def rollback( source_format=TARGET_FORMAT_VERSION, migration_run_id=run_id, active_generation=pointer.generation, + warnings=( + ( + f"{hidden} event(s) recorded since activation stay only in " + f"{current_relative}", + ) + if hidden + else () + ), ) From 02fa2b0010fa25aad55b6c6297e04b0c79e8ee64 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:48:25 -0400 Subject: [PATCH 51/72] fix: stop legacy commands upgrading the database migrate-v7 rolls back to `cli status` on a v6 store ran the whole migration ledger in place -- schema 15 to 32, no backup, no pointer -- which rewrote the file `migrate-v7` snapshots and can roll back to, and left its governance backfill to go stale against later v6 edits. `init_db` now caps a format-6 store at the last v6 schema version, and `_validate_database` requires that version rather than the current one for format 6. Co-Authored-By: Claude Opus 5 --- daem0nmcp/database.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/daem0nmcp/database.py b/daem0nmcp/database.py index 93c2a07..d975e4d 100644 --- a/daem0nmcp/database.py +++ b/daem0nmcp/database.py @@ -209,6 +209,15 @@ def _run_migrations( self._migrated = True + def _v6_migration_ceiling(self) -> int | None: + """The migration ceiling for a retained format-6 store, else ``None``.""" + + if self.format_version != 6: + return None + from .migrations.schema import _LAST_V6_SCHEMA_VERSION + + return _LAST_V6_SCHEMA_VERSION + async def init_db(self): """Initialize the database tables and run migrations.""" # Skip if already initialized @@ -224,7 +233,11 @@ async def init_db(self): # Run migrations first for existing databases (sync operation) # This happens BEFORE we create the async engine to avoid lock conflicts if not is_new_db: - self._run_migrations() + # A format-6 store is what `migrate-v7` snapshots and can roll back + # to. Applying the v7 ledger to it in place would rewrite the + # source without a backup, and its governance backfill then goes + # stale against later v6 edits, so cap it at the last v6 version. + self._run_migrations(maximum_version=self._v6_migration_ceiling()) # Then create any new tables async with self.engine.begin() as conn: @@ -425,7 +438,14 @@ def _validate_database(self, *, format_version: int) -> None: foreign = list(connection.execute("PRAGMA foreign_key_check")) if integrity != ["ok"] or foreign: raise RuntimeError("DATABASE_INTEGRITY_FAILED") - if self._schema_version() < CURRENT_SCHEMA_VERSION: + from .migrations.schema import _LAST_V6_SCHEMA_VERSION + + # A retained format-6 store stops at the last v6 version on + # purpose: `migrate-v7` owns everything above it. + if format_version == 7: + if self._schema_version() < CURRENT_SCHEMA_VERSION: + raise RuntimeError("SCHEMA_MIGRATION_INCOMPLETE") + elif self._schema_version() < _LAST_V6_SCHEMA_VERSION: raise RuntimeError("SCHEMA_MIGRATION_INCOMPLETE") if format_version == 7: required = { From 06bdd1e92a64ef228159d31233d4c8d20cc1e3c1 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:48:25 -0400 Subject: [PATCH 52/72] feat: tell an un-migrated workspace what to run An intact v6 store answered CAPABILITY_DEGRADED, retryable, with no remedy, although retrying can never help. `MIGRATION_REQUIRED` is non-retryable and its message names the command, with a placeholder instead of a host path and the instruction to stop the server first. The briefing, the pinned services and the core operations all raise it for a format-6 active database. `migrate-v7 --rollback` gains `--discard-v7-writes` for the refusal above. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/errors.py | 11 +++++++++++ daem0nmcp/api/v7/pinned.py | 7 ++++++- daem0nmcp/api/v7/production.py | 6 ++++++ daem0nmcp/cli.py | 14 +++++++++++++- tests/api_v7/test_models.py | 1 + tests/test_migrate_v7.py | 12 +++++++++--- 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/daem0nmcp/api/v7/errors.py b/daem0nmcp/api/v7/errors.py index c42fe14..c6a0312 100644 --- a/daem0nmcp/api/v7/errors.py +++ b/daem0nmcp/api/v7/errors.py @@ -22,6 +22,7 @@ class ErrorCode(str, Enum): STALE_PROJECTION_ID = "STALE_PROJECTION_ID" CAPABILITY_DISABLED = "CAPABILITY_DISABLED" CAPABILITY_DEGRADED = "CAPABILITY_DEGRADED" + MIGRATION_REQUIRED = "MIGRATION_REQUIRED" LEXICAL_UNAVAILABLE = "LEXICAL_UNAVAILABLE" COMMUNION_REQUIRED = "COMMUNION_REQUIRED" COUNSEL_REQUIRED = "COUNSEL_REQUIRED" @@ -56,6 +57,15 @@ class ErrorCode(str, Enum): # caller; server-side logs may contain the corresponding private details. INTERNAL_ERROR_MESSAGE = "Internal error." +# A retained v6 store cannot be read by v7 and retrying never helps. The +# remedy is an offline command, not an MCP tool, so it is stated in the message +# with a placeholder instead of a machine-readable ``remedy``. +MIGRATION_REQUIRED_MESSAGE = ( + "This workspace still uses the v6 storage format. Stop the server, then " + "run: python -m daem0nmcp.cli --project-path migrate-v7 " + "--apply" +) + def is_stable_error_code(value: object) -> bool: """Return whether *value* is an exact reviewed v7 error code.""" @@ -66,6 +76,7 @@ def is_stable_error_code(value: object) -> bool: __all__ = [ "ERROR_CODE_REGISTRY", "INTERNAL_ERROR_MESSAGE", + "MIGRATION_REQUIRED_MESSAGE", "STABLE_ERROR_CODES", "STABLE_ERROR_CODE_SET", "ErrorCode", diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 1a873ef..75d4d4e 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -19,7 +19,7 @@ from ...event_store import AppendedEvent, EventStreamConflict from ...retrieval import RetrievalQuery from ...workspace import Workspace -from .errors import ErrorCode +from .errors import MIGRATION_REQUIRED_MESSAGE, ErrorCode from .models import ApiResponse, CapabilityState, RecordSummary, RetrievalData from .responses import ResponseContext, ResponseFactory from .tasks import ( @@ -114,6 +114,11 @@ def __init__(self, code: str) -> None: "The active v7 workspace is unavailable.", True, ), + "MIGRATION_REQUIRED": ( + ErrorCode.MIGRATION_REQUIRED, + MIGRATION_REQUIRED_MESSAGE, + False, + ), "FEDERATION_UNAVAILABLE": ( ErrorCode.CAPABILITY_DISABLED, "Linked-workspace retrieval is unavailable.", diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 498cd4a..3dc54a6 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -336,6 +336,10 @@ def _capability_states( def _active_database(workspace: Workspace): storage = resolve_workspace_storage(workspace) active = resolve_active_database(storage) + if active.format_version == 6: + # The store is intact but still v6; retrying never helps, so say so + # and name the offline command that migrates it. + raise ProductionConfigurationError("MIGRATION_REQUIRED") if active.format_version != 7: raise ProductionConfigurationError("ACTIVE_V7_UNAVAILABLE") return active @@ -460,6 +464,8 @@ async def read( ) -> dict[str, object]: try: _active_database(workspace) + except ProductionConfigurationError as exc: + raise RuntimeServiceError(exc.code) from None except Exception: raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None warning_limit = request.warning_limit diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..b294e2a 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -352,6 +352,14 @@ def main(): metavar="MIGRATION_ID", help="Roll back the active migration run (default: latest)", ) + migrate_v7_parser.add_argument( + "--discard-v7-writes", + action="store_true", + help=( + "Roll back even though events recorded after activation would " + "become unreachable" + ), + ) migrate_v7_parser.add_argument( "--batch-size", type=_v7_batch_size, @@ -637,7 +645,11 @@ def main(): service = MigrationV7Service(registry) try: if args.rollback is not None: - result = service.rollback(args.project_path, args.rollback) + result = service.rollback( + args.project_path, + args.rollback, + discard_v7_writes=args.discard_v7_writes, + ) elif args.apply: result = service.apply(args.project_path, batch_size=args.batch_size) else: diff --git a/tests/api_v7/test_models.py b/tests/api_v7/test_models.py index 511e0fa..4d2e78a 100644 --- a/tests/api_v7/test_models.py +++ b/tests/api_v7/test_models.py @@ -70,6 +70,7 @@ def test_error_registry_is_exact_and_rejects_unknown_codes(self) -> None: "STALE_PROJECTION_ID", "CAPABILITY_DISABLED", "CAPABILITY_DEGRADED", + "MIGRATION_REQUIRED", "LEXICAL_UNAVAILABLE", "COMMUNION_REQUIRED", "COUNSEL_REQUIRED", diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index cb4969d..33cc06c 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -1430,7 +1430,9 @@ def test_reactivation_validates_evolved_v7_authority_not_original_v6_rows(self): candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) reactivated = service.apply(root) self.assertEqual("reactivate", reactivated.action) @@ -1500,7 +1502,9 @@ def test_reactivation_rejects_current_memory_compatibility_divergence(self): ) candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) candidate = sqlite3.connect(candidate_path) candidate.execute( @@ -1621,7 +1625,9 @@ def test_reactivation_rejects_ambiguous_live_compatibility_claims(self): ) candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) with self.assertRaisesRegex(MigrationV7Error, "VALIDATION_FAILED"): service.apply(root) From 15601c456beae22099e770050cde0b56bc8f6b46 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 01:48:54 -0400 Subject: [PATCH 53/72] test: prove a migrated v6 workspace is usable end to end The fixture is a chain-built format-6 store -- the retained v6 ORM tables plus the real migration ledger capped at the last v6 version -- seeded with rows shaped as v6 `remember` wrote them. Its column layout was compared against a database produced by running the v6 code at commit 00809c6 and matches column for column, so this is not a hand-written table set (F-053). Through the real subprocess server: migrate, `session_brief` with the migrated rule, `memory_recall` returning the file-linked record with its relative path, `memory_record_outcome`, and a complete `workspace_export` whose legacy rows carry no host path. Around it: every migrated row reads back through every reader; a store left in the pre-fix shape still reads; a moved project keeps its file links; a sibling directory is not inside the workspace; rollback refuses to hide post-activation writes and `--discard-v7-writes` leaves the v6 file byte-identical; an un-migrated store answers MIGRATION_REQUIRED; a v6 rule edited after an in-place upgrade reaches v7; a foreign trigger is counted; `cli status` leaves the v6 store at version 15. Export pages come from one `connection.backup()` snapshot written to disk when the session opens, so a write between pages cannot change a later one; the test pins that rather than assuming it. Co-Authored-By: Claude Opus 5 --- tests/test_migrate_v7_end_to_end.py | 697 ++++++++++++++++++++++++++++ 1 file changed, 697 insertions(+) create mode 100644 tests/test_migrate_v7_end_to_end.py diff --git a/tests/test_migrate_v7_end_to_end.py b/tests/test_migrate_v7_end_to_end.py new file mode 100644 index 0000000..df97c13 --- /dev/null +++ b/tests/test_migrate_v7_end_to_end.py @@ -0,0 +1,697 @@ +"""A migrated v6 workspace must be usable: brief, recall, outcome, export. + +The fixture is built the way v6 built its own database -- the retained v6 ORM +tables plus the real migration ledger capped at ``_LAST_V6_SCHEMA_VERSION`` -- +and seeded with rows shaped exactly as v6 ``remember`` wrote them, including +the host-absolute ``file_path``, the ``../outside`` relative form v6 writes for +a file above the project, unbounded tags and empty content. The column +layout was compared against a database produced by running the v6 code at +commit 00809c6; it matches column for column. +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +import sys +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client +from sqlalchemy import create_engine + +from daem0nmcp import models +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.config import Settings +from daem0nmcp.migrations.schema import _LAST_V6_SCHEMA_VERSION, run_migrations +from daem0nmcp.migrations.v7 import MigrationV7Error, MigrationV7Service +from daem0nmcp.storage_activation import resolve_active_database +from daem0nmcp.workspace import ( + WorkspaceRegistry, + is_workspace_relative_path, + relative_to_root, +) +from tests.api_v7.process_client import call, process_client + +V6_TABLES = ( + "memories", + "memory_versions", + "facts", + "memory_relationships", + "rules", + "context_triggers", + "active_context", +) +PROFILE_ENVIRONMENT = {"DAEM0NMCP_PROFILE": "core"} +TIME = "2026-01-01 00:00:00" + + +def _v6_file_paths(file_path: Path, project_path: Path) -> tuple[str, str]: + """Reproduce v6 ``_normalize_file_path`` (v6 memory.py:63-106) exactly.""" + + resolved = file_path.resolve() + absolute = str(resolved) + root = project_path.resolve() + try: + relative = resolved.relative_to(root).as_posix() + except ValueError: + import os.path + + relative = os.path.relpath(resolved, start=root).replace("\\", "/") + if sys.platform == "win32": + absolute = absolute.lower() + relative = relative.lower() + return absolute, relative + + +def _v6_database(root: Path, *, project_path: Path | None = None) -> Path: + """Build and seed a format-6 store below ``root`` and return its storage.""" + + project = project_path or root + storage = root / ".daem0nmcp" / "storage" + storage.mkdir(parents=True) + database = storage / "daem0nmcp.db" + engine = create_engine(f"sqlite:///{database}") + models.Base.metadata.create_all( + engine, tables=[models.Base.metadata.tables[name] for name in V6_TABLES] + ) + engine.dispose() + run_migrations(str(database), maximum_version=_LAST_V6_SCHEMA_VERSION) + + inside_absolute, inside_relative = _v6_file_paths( + project / "src" / "db.py", project + ) + route_absolute, route_relative = _v6_file_paths(project / "src" / "app.py", project) + outside_absolute, outside_relative = _v6_file_paths( + project.parent / "other" / "helper.py", project + ) + memories = ( + ( + 1, + "decision", + "Use postgres for the analytics warehouse", + "SQLite could not keep up with the nightly rollups", + '{"ticket": "ANA-12"}', + '["database", "postgres"]', + inside_absolute, + inside_relative, + 0, + 0, + ), + ( + 2, + "warning", + "Keep the /health endpoint unauthenticated; crash was in C:\\proj\\app.py", + "Load balancer probes it", + "{}", + '["http"]', + route_absolute, + route_relative, + 1, + 0, + ), + ( + 3, + "pattern", + "The shared helper lives outside the project", + None, + "{}", + '["shared"]', + outside_absolute, + outside_relative, + 1, + 0, + ), + ( + 4, + "learning", + "Migrated rows must stay readable", + None, + "{}", + json.dumps( + ["x" * 120, "dup", "dup", "ctrl\x01char", *[f"t{n}" for n in range(40)]] + ), + None, + None, + 0, + 1, + ), + (5, "learning", "", None, "{}", "[]", None, None, 0, 0), + ) + connection = sqlite3.connect(database) + try: + for row in memories: + connection.execute( + "INSERT INTO memories (id,category,content,rationale,context,tags," + "file_path,file_path_relative,keywords,is_permanent,vector_embedding," + "outcome,worked,pinned,archived,recall_count,surprise_score," + "importance_score,source_client,source_model,created_at,updated_at) " + "VALUES (?,?,?,?,?,?,?,?,'kw',?,NULL,NULL,NULL,?,0,0,NULL,NULL," + "'claude-code','sonnet',?,?)", + (*row, TIME, TIME), + ) + connection.execute( + "UPDATE memories SET outcome='Cut the rollup to 4 minutes',worked=1 " + "WHERE id=1" + ) + connection.execute( + "INSERT INTO memory_versions (id,memory_id,version_number,content," + "rationale,context,tags,outcome,worked,change_type,change_description," + "changed_at,valid_from,valid_to,invalidated_by_version_id) " + "VALUES (1,1,1,'Use postgres',NULL,'{}','[\"database\"]',NULL,NULL," + "'created','initial',?,?,NULL,NULL)", + (TIME, TIME), + ) + connection.execute( + "INSERT INTO facts (id,content_hash,content,category,source_memory_id," + "verification_count,is_verified,tags,created_at,verified_at) " + "VALUES (1,'legacy-hash','Postgres is remote','database',1,2,1," + "'[\"db\"]',?,NULL)", + (TIME,), + ) + connection.execute( + "INSERT INTO memory_relationships (id,source_id,target_id,relationship," + "description,confidence,created_at) VALUES " + "(1,1,2,'related_to','both touch the API tier',1.0,?)", + (TIME,), + ) + connection.execute( + "INSERT INTO rules (id,trigger,trigger_keywords,must_do,must_not," + "ask_first,warnings,priority,enabled,created_at) VALUES " + "(1,'before editing the schema','schema','[\"write a migration\"]'," + "'[\"drop tables\"]','[]','[]',5,1,?)", + (TIME,), + ) + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(1,?,'file_pattern','src/*.py','source conventions'," + "'[\"decision\"]',1,3,?,0,NULL)", + (str(project), TIME), + ) + connection.execute( + "INSERT INTO active_context (id,project_path,memory_id,priority,reason," + "added_at,expires_at) VALUES (1,?,1,4,'migration focus',?,NULL)", + (str(project), TIME), + ) + connection.commit() + finally: + connection.close() + return storage + + +def _registry(root: Path) -> WorkspaceRegistry: + return WorkspaceRegistry([root], default_root=root) + + +def _workspace_id(root: Path) -> str: + return _registry(root).default.workspace_id + + +def _server(root: Path): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(root), + workspace_roots=[str(root)], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, + ) + + +def _records(storage: Path) -> list[sqlite3.Row]: + connection = sqlite3.connect(resolve_active_database(storage).path) + connection.row_factory = sqlite3.Row + try: + return connection.execute( + "SELECT * FROM memory_records ORDER BY created_at_us,record_id" + ).fetchall() + finally: + connection.close() + + +async def _invoke(client: Client, workspace_id: str, tool: str, **arguments: Any): + result = await client.call_tool( + tool, {"workspace_id": workspace_id, **arguments}, raise_on_error=False + ) + assert result.structured_content is not None, tool + return result.structured_content + + +async def _succeed(client: Client, workspace_id: str, tool: str, **arguments: Any): + response = await _invoke(client, workspace_id, tool, **arguments) + assert response["ok"], (tool, response["error"]) + return response["data"] + + +def test_v6_fixture_uses_the_real_migration_ledger(tmp_path): + """The fixture is a chain-built v6 store, not a hand-written table set.""" + + storage = _v6_database(tmp_path / "project") + connection = sqlite3.connect(storage / "daem0nmcp.db") + try: + versions = [ + int(row[0]) + for row in connection.execute("SELECT version FROM schema_version") + ] + finally: + connection.close() + assert max(versions) == _LAST_V6_SCHEMA_VERSION + assert set(versions) >= { + version + for version, _, _ in __import__( + "daem0nmcp.migrations.schema", fromlist=["MIGRATIONS"] + ).MIGRATIONS + if version <= _LAST_V6_SCHEMA_VERSION + } + + +@pytest.mark.parametrize("transport", ["stdio"]) +async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path, transport): + """F-003/F-004/F-007/F-008/F-053: migrate, brief, recall, outcome, export.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.status == "activated", result + assert result.validation["rule_count"] == 1 + assert result.validation["trigger_count"] == 1 + # The one memory whose file lives above the project cannot keep a + # workspace-relative link, and the migration says so instead of writing a + # path that later fails every read. + assert result.validation["dropped_file_link_count"] == 1 + assert any("file link" in warning for warning in result.warnings) + + rows = {row["content"]: row for row in _records(storage)} + assert all(row["file_path"] is None for row in rows.values()) + assert rows["Use postgres for the analytics warehouse"]["file_path_relative"] == ( + "src/db.py" + ) + # The helper lives above the project: no workspace-relative link exists. + assert ( + rows["The shared helper lives outside the project"]["file_path_relative"] + is None + ) + # v6 tags had no length, uniqueness or count limit; the v7 wire has all three. + tags = json.loads(rows["Migrated rows must stay readable"]["tags_json"]) + assert len(tags) == 32 and len(set(tags)) == 32 + assert all(len(tag) <= 80 for tag in tags) + assert "" in rows, sorted(rows) + + workspace_id = _workspace_id(root) + async with process_client(root, transport) as session: + brief = await call(session, "session_brief", {"workspace_id": workspace_id}) + assert brief["ok"], brief["error"] + assert brief["data"]["applicable_rules"], "the migrated v6 rule must brief" + + recall = await call( + session, + "memory_recall", + {"workspace_id": workspace_id, "query": "postgres"}, + ) + assert recall["ok"], recall["error"] + linked = [ + item["record"] + for item in recall["data"]["items"] + if item["record"]["relative_file_path"] == "src/db.py" + ] + assert linked, recall["data"]["items"] + + outcome = await call( + session, + "memory_record_outcome", + { + "workspace_id": workspace_id, + "record_id": linked[0]["record_id"], + "outcome_text": "still true after the migration", + "worked": True, + "idempotency_key": "migrated-outcome-0001", + }, + ) + assert outcome["ok"], outcome["error"] + + pages: list[dict[str, Any]] = [] + arguments: dict[str, Any] = {} + while len(pages) <= 32: + page = await call( + session, + "workspace_export", + {"workspace_id": workspace_id, **arguments}, + ) + assert page["ok"], page["error"] + pages.append(page["data"]) + if page["data"]["complete"] or page["data"]["next_cursor"] is None: + break + arguments = { + "export_session_id": page["data"]["export_session_id"], + "cursor": page["data"]["next_cursor"], + "page_index": page["data"]["page_index"] + 1, + } + assert pages and pages[-1]["complete"] + # Exporting at all proves no host path reached a structured path field + # (the guard refuses the whole page); assert the surviving links too. + exported = [row for page in pages for row in page["legacy_rows"]] + assert exported + for row in exported: + relative = row["record"]["file_path_relative"] + assert relative is None or is_workspace_relative_path(relative), relative + assert "file_path" not in row["record"] + + +async def test_every_migrated_row_reads_back_through_the_server(tmp_path): + """A migrated row cannot deny a read, whatever v6 put in it.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + for query in ("postgres", "health", "readable", "helper"): + await _succeed(client, workspace_id, "memory_recall", query=query) + await _succeed(client, workspace_id, "memory_search_text", query=query) + for kind in ("warnings", "failures", "rules", "active-context"): + await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") + + +async def test_a_database_migrated_by_the_pre_fix_code_still_reads(tmp_path): + """The values the previous migration wrote must not brick a read.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + # Put the projection back into the shape the pre-fix migration left: the + # host-absolute file_path, v6's ``../`` relative form, v6's unbounded tags + # and empty content. No re-migration is offered for such a store. + absolute, relative = _v6_file_paths(tmp_path / "other" / "helper.py", root) + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + connection.execute( + "UPDATE memory_records SET file_path=?,file_path_relative=?,tags_json=?," + "content=? WHERE record_type='pattern'", + ( + absolute, + relative, + json.dumps(["y" * 120, "same", "same", *[f"u{n}" for n in range(40)]]), + "", + ), + ) + connection.commit() + finally: + connection.close() + + async with Client(_server(root)) as client: + brief = await _succeed(client, workspace_id, "session_brief") + assert isinstance(brief["warnings"], list) + recall = await _succeed(client, workspace_id, "memory_recall", query="helper") + for item in recall["items"]: + assert item["record"]["relative_file_path"] != relative + for kind in ("warnings", "failures", "rules", "active-context"): + await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") + + +async def test_a_moved_project_keeps_its_file_links(tmp_path): + """F-022: v6 recorded one project root; file links resolve against it.""" + + old_root = tmp_path / "old-name" + (old_root / "src").mkdir(parents=True) + new_root = tmp_path / "new-name" + (tmp_path / "other").mkdir() + storage = _v6_database(old_root, project_path=old_root) + # v6 kept the database inside the project, so moving the directory moves + # the store with it while every recorded path still names the old root. + shutil.move(str(old_root), str(new_root)) + storage = new_root / ".daem0nmcp" / "storage" + + result = MigrationV7Service(_registry(new_root)).apply(new_root) + assert result.status == "activated", result + # The single recorded v6 project path is this workspace's own history. + assert result.validation["trigger_count"] == 1 + assert result.validation["foreign_trigger_count"] == 0 + rows = {row["record_type"]: row for row in _records(storage)} + assert rows["decision"]["file_path_relative"] == "src/db.py" + + workspace_id = _workspace_id(new_root) + async with Client(_server(new_root)) as client: + await _succeed(client, workspace_id, "session_brief") + await _succeed(client, workspace_id, "memory_recall", query="postgres") + + +def test_a_sibling_directory_is_not_inside_the_workspace(tmp_path): + """R-25: containment is decided by path parts, never a string prefix.""" + + inside = tmp_path / "proj" + sibling = tmp_path / "proj2" + (inside / "src").mkdir(parents=True) + sibling.mkdir() + (inside / "src" / "a.py").write_text("", encoding="utf-8") + (sibling / "a.py").write_text("", encoding="utf-8") + + assert relative_to_root(inside / "src" / "a.py", inside) == "src/a.py" + assert relative_to_root(sibling / "a.py", inside) is None + assert relative_to_root(inside, inside) is None + assert relative_to_root(tmp_path / "proj" / ".." / "proj2" / "a.py", inside) is None + + +async def test_rollback_refuses_to_hide_writes_made_since_activation(tmp_path): + """F-024/F-068: rollback is loud about the v7 events it would strand.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + source_bytes = (storage / "daem0nmcp.db").read_bytes() + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written after the migration", + "idempotency_key": "post-migration-0001", + }, + description="write after migrating", + ) + stored = await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written after the migration", + idempotency_key="post-migration-0001", + preflight_token=preflight["preflight_token"], + ) + assert stored["record"]["record_id"] + + with pytest.raises(MigrationV7Error) as refusal: + service.rollback(root) + assert refusal.value.code == "ROLLBACK_WOULD_HIDE_WRITES" + assert resolve_active_database(storage).format_version == 7 + + result = service.rollback(root, discard_v7_writes=True) + assert result.status == "rolled_back" + assert result.warnings and "unreachable" not in result.warnings[0] + assert resolve_active_database(storage).format_version == 6 + assert (storage / "daem0nmcp.db").read_bytes() == source_bytes + + +async def test_an_un_migrated_v6_workspace_says_migration_is_required(tmp_path): + """F-066/F-069/F-043: a non-retryable error naming the migrate command.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + response = await _invoke(client, workspace_id, "session_brief") + assert response["ok"] is False + assert response["error"]["code"] == "MIGRATION_REQUIRED" + assert response["error"]["retryable"] is False + assert "migrate-v7 --apply" in response["error"]["message"] + + +def test_a_v6_rule_edited_after_an_in_place_upgrade_reaches_v7(tmp_path): + """F-023: a `must_not` added after an in-place upgrade must not vanish.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + database = storage / "daem0nmcp.db" + # The pre-fix legacy CLI applied the whole ledger to the v6 file, which + # backfilled governance from the rules table as it stood then. + run_migrations(str(database), workspace_id=_workspace_id(root)) + connection = sqlite3.connect(database) + try: + connection.execute( + "UPDATE rules SET must_not=? WHERE id=1", + (json.dumps(["drop tables", "truncate"]),), + ) + connection.execute("DELETE FROM context_triggers WHERE id=1") + connection.commit() + finally: + connection.close() + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.status == "activated", result + assert result.validation["rules_updated"] == 1 + assert result.validation["triggers_deleted"] == 1 + + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + must_not = connection.execute( + "SELECT must_not_json FROM governance_rules" + ).fetchone()[0] + deleted = connection.execute( + "SELECT count(*) FROM governance_context_triggers " + "WHERE deleted_at_us IS NOT NULL" + ).fetchone()[0] + finally: + connection.close() + assert json.loads(must_not) == ["drop tables", "truncate"] + assert deleted == 1 + + +def test_a_foreign_project_trigger_is_skipped_and_counted(tmp_path): + """F-022: triggers from other project roots are reported, not dropped.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + connection = sqlite3.connect(storage / "daem0nmcp.db") + try: + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(2,?,'tag_match','auth.*','authentication','[\"warning\"]',1,1,?,0,NULL)", + (str(tmp_path / "another-project"), TIME), + ) + connection.commit() + finally: + connection.close() + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.validation["trigger_count"] == 1 + assert result.validation["foreign_trigger_count"] == 1 + assert any("another project path" in warning for warning in result.warnings) + + +async def test_export_pages_come_from_one_snapshot(tmp_path): + """Writes between pages cannot change a page already in the session.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + first = await _succeed( + client, workspace_id, "workspace_export", page_byte_limit=65_536 + ) + assert first["page_count"] >= 2, "the fixture must page" + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written between export pages", + "idempotency_key": "between-pages-0001", + }, + description="write between export pages", + ) + await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written between export pages", + idempotency_key="between-pages-0001", + preflight_token=preflight["preflight_token"], + ) + page = first + collected = [first] + while not page["complete"] and page["next_cursor"] is not None: + page = await _succeed( + client, + workspace_id, + "workspace_export", + export_session_id=page["export_session_id"], + cursor=page["next_cursor"], + page_index=page["page_index"] + 1, + ) + collected.append(page) + assert page["complete"] + # Every page belongs to the session's manifest, taken before the write. + assert {item["manifest_hash"] for item in collected} == {first["manifest_hash"]} + assert all(item["root_hash"] == first["root_hash"] for item in collected), ( + "a later page must not observe an event written after the session" + ) + + +def test_cli_status_leaves_a_v6_database_at_its_last_v6_version(tmp_path): + """F-023/A-005: a legacy command must not upgrade the migration source.""" + + import subprocess + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + database = storage / "daem0nmcp.db" + + completed = subprocess.run( + [ + sys.executable, + "-m", + "daem0nmcp.cli", + "--project-path", + str(root), + "--json", + "status", + ], + capture_output=True, + text=True, + timeout=300, + ) + assert completed.returncode == 0, completed.stderr[-2000:] + connection = sqlite3.connect(database) + try: + version = connection.execute( + "SELECT MAX(version) FROM schema_version" + ).fetchone()[0] + finally: + connection.close() + assert version == _LAST_V6_SCHEMA_VERSION + assert resolve_active_database(storage).format_version == 6 + # The source is still migratable afterwards. + assert MigrationV7Service(_registry(root)).apply(root).status == "activated" From bbf734157ee7202e6c9775f08b8443602e287e8b Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 02:00:09 -0400 Subject: [PATCH 54/72] docs: say that the v6-to-v7 format migration is an explicit command The README told an upgrading v6 user that migrations are automatic and no manual step is required, which is true only within a storage format. Both documents now state the offline command, that the server must be stopped first, that a v6 workspace answers MIGRATION_REQUIRED until it runs, that rollback refuses to hide post-activation writes, and what `warnings` reports about rows the migration could not copy faithfully. Co-Authored-By: Claude Opus 5 --- README.md | 18 ++++++++++++++++-- docs/v7-operations.md | 21 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index daad047..bf46450 100644 --- a/README.md +++ b/README.md @@ -1038,9 +1038,23 @@ python -m daem0nmcp.cli install-opencode --force This regenerates `opencode.json` and the plugin to match the latest version. -### 5. Migrations Run Automatically +### 5. Migrations -Database schema migrations are applied automatically when any MCP tool runs. No manual migration step required. +Within a storage format, schema migrations are applied automatically when any +MCP tool runs, and no manual step is required. + +Moving an existing v6 workspace to the v7 storage format is **not** automatic. +It is an offline, reversible command, and a v6 workspace answers +`MIGRATION_REQUIRED` until it has run: + +```bash +# Stop the server first, then: +python -m daem0nmcp.cli --project-path /path/to/project migrate-v7 --apply + +# Undo it; refuses if events were recorded since activation unless +# --discard-v7-writes is passed. +python -m daem0nmcp.cli --project-path /path/to/project migrate-v7 --rollback +``` ### 6. Index Your Codebase diff --git a/docs/v7-operations.md b/docs/v7-operations.md index ba19798..57ee845 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -197,9 +197,24 @@ the dry-run inventory. `FUTURE_V7_SCHEMA`, `UNSUPPORTED_V7_SCHEMA`, and Daem0nMCP build; they are not repaired automatically. After an interrupted run, repeat the same `migrate-v7 --apply` command. Use `migrate-v7 --rollback latest` to return to the retained predecessor and `--apply` to reactivate its verified -candidate. The generic `migrate` command remains an in-place additive migration -entry point; use `migrate-v7 --apply` when retention, verification, atomic -activation, and rollback are required. +candidate. Rollback refuses with `ROLLBACK_WOULD_HIDE_WRITES`, naming the count +and the candidate that retains them, when events were recorded after +activation; `--discard-v7-writes` proceeds and reports what stayed behind. The +generic `migrate` command remains an in-place additive migration entry point, +capped at the last v6 schema version while a store is still format 6 so it +cannot rewrite what `migrate-v7` snapshots; use `migrate-v7 --apply` when +retention, verification, atomic activation, and rollback are required. + +Stop the server before applying or rolling back: both commands take the +storage lock, and a running server on the old pointer sees the generation +change. Until a v6 workspace is migrated, every tool answers +`MIGRATION_REQUIRED` (non-retryable) with the command to run. + +`migrate-v7 --apply` reports what it could not copy faithfully in `warnings` +and counts it in `validation`: memory file links that pointed outside the +workspace (kept only in the event's legacy payload), context triggers +belonging to another project path, and rules or triggers reconciled because +the v6 tables changed after an earlier in-place upgrade. `verify-v7` checks authority and projections; `--repair-projections` performs offline replay and atomic activation. Rollback retains the v7 write history From 0d9b0045e032f90fe72b492dd6dbf6c7bd5ce2f2 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 02:08:07 -0400 Subject: [PATCH 55/72] refactor: delete the drive regex RelativePath no longer uses `is_workspace_relative_path` is now the single implementation behind `RelativePath`, so the copy in models.py had no remaining caller. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index 998bee1..55d6d09 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -49,7 +49,6 @@ r"[0-9]{2}:[0-9]{2}:[0-9]{2}" r"(?:\.[0-9]{1,9})?(?:Z|[+-][0-9]{2}:[0-9]{2})$" ) -_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") _WINDOWS_ABSOLUTE_PATH = re.compile( r"(? Date: Sun, 20 Sep 2026 02:40:15 -0400 Subject: [PATCH 56/72] docs: state the dry-run guarantee as PR 1 left it On POSIX a source with a WAL sidecar is read through the default VFS, whose volatile ``-shm`` write the comment already describes, so the opening sentence no longer claims no byte is written anywhere. Co-Authored-By: Claude Opus 5 --- daem0nmcp/migrations/v7.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index 38bc211..2426d3c 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -162,7 +162,7 @@ def _quoted_identifier(name: str) -> str: def _readonly_connection(path: Path) -> sqlite3.Connection: - # Migration dry-run must not write any byte. On Windows the no-lock VFS + # A dry run never changes the source database. On Windows the no-lock VFS # keeps read-only connections off the ``-shm`` read marks. POSIX has no # equivalent: ``unix-none`` lacks shared memory, so it cannot open a WAL # database at all. There, with no ``-wal``, ``-shm`` or ``-journal`` the From 49472682867b9856f5ac939a379a606acc9467e8 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 05:16:12 -0400 Subject: [PATCH 57/72] test: wait on the fact, not on a duration, in two timing-sensitive tests Both failed on the slowest Python 3.10 CI legs of run 35495906995 and neither touches this PR's code. `test_write_locked_builder_outlasts_heartbeat_busy_timeout` held the writer lock for a fixed 1.25 s against a 100 ms lease, so its whole margin was the 100 ms between the heartbeat's final refresh and the lease check in `_record_success`; a descheduled runner lost it and the job that had in fact completed was recorded as PROJECTION_JOB_LEASE_LOST. The builder now holds the lock until the heartbeat has provably been through its busy timeout more than once (counted on a connection subclass, so real contention is asserted rather than assumed) and the lease it was claimed under has expired, both to a bounded deadline, and the lease is long enough that the terminal update cannot lose a race it was never about. Every assertion is unchanged and one is added. Proved on python:3.10-slim by injecting a 150 ms stall into exactly that window: the previous test fails 5/5 with the CI's error, the new one passes 5/5, and the previous test passes without the stall. `test_production_ritual_and_restart` polled to a deadline for the record to become visible before the restart but asserted immediately after it, although a restarted server republishes the lexical generation in the background. Both now use one bounded poll. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_process_client.py | 28 +++++++++++----- tests/test_retrieval_jobs.py | 51 +++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/tests/api_v7/test_process_client.py b/tests/api_v7/test_process_client.py index 8f71340..0b30564 100644 --- a/tests/api_v7/test_process_client.py +++ b/tests/api_v7/test_process_client.py @@ -151,13 +151,7 @@ async def test_production_ritual_and_restart(initialized_workspace, transport): "idempotency_key": "process-acceptance-outcome-0001", }, ) - visibility_deadline = asyncio.get_running_loop().time() + 5 - while True: - recalled = await succeed(session, "memory_recall", recall_arguments) - if record_id in {item["record"]["record_id"] for item in recalled["items"]}: - break - assert asyncio.get_running_loop().time() < visibility_deadline, recalled - await asyncio.sleep(0.1) + await _recall_until_visible(session, recall_arguments, record_id) templates = await session.list_resource_templates() assert len(templates.resourceTemplates) >= 4 resource = await session.read_resource( @@ -172,8 +166,24 @@ async def test_production_ritual_and_restart(initialized_workspace, transport): denied = await call(restarted, "memory_recall", recall_arguments) assert denied["error"]["code"] == "COMMUNION_REQUIRED" await succeed(restarted, "session_brief", scope) - recalled = await succeed(restarted, "memory_recall", recall_arguments) - assert record_id in {item["record"]["record_id"] for item in recalled["items"]} + # A restarted server republishes the lexical generation in the + # background, so the first recall after it can still read the + # previous one. Poll to a bounded deadline instead of racing it. + await _recall_until_visible(restarted, recall_arguments, record_id) + + +async def _recall_until_visible( + session, recall_arguments: dict, record_id: str, *, timeout: float = 30 +) -> None: + """Recall until the record is visible, or fail at a bounded deadline.""" + + deadline = asyncio.get_running_loop().time() + timeout + while True: + recalled = await succeed(session, "memory_recall", recall_arguments) + if record_id in {item["record"]["record_id"] for item in recalled["items"]}: + return + assert asyncio.get_running_loop().time() < deadline, recalled + await asyncio.sleep(0.1) async def test_federated_recall_process_requires_each_briefing_and_keeps_origin( diff --git a/tests/test_retrieval_jobs.py b/tests/test_retrieval_jobs.py index 9bda81e..1b5e173 100644 --- a/tests/test_retrieval_jobs.py +++ b/tests/test_retrieval_jobs.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import itertools import json import sqlite3 import tempfile +import threading import time import unittest from pathlib import Path @@ -391,13 +393,53 @@ def test_write_locked_builder_outlasts_heartbeat_busy_timeout(self): from daem0nmcp.retrieval.jobs import ProjectionJobRunner self._queue_job() + lease_duration_us = 1_000_000 + contended = threading.Event() + contentions = itertools.count(1) + + class _CountingConnection(sqlite3.Connection): + """Report when the heartbeat has really hit its busy timeout.""" + + def execute(self, sql, parameters=()): # type: ignore[override] + try: + return super().execute(sql, parameters) + except sqlite3.OperationalError: + if sql.startswith("BEGIN EXCLUSIVE") and next(contentions) >= 2: + contended.set() + raise + + def heartbeat_connection(): + return sqlite3.connect( + self.database_path, timeout=1.0, factory=_CountingConnection + ) def write_locked_build(_workspace_id): self.connection.execute("BEGIN IMMEDIATE") self.connection.execute( "UPDATE background_jobs SET result_json=result_json" ) - time.sleep(1.25) + claimed_lease_us = int( + self.connection.execute( + "SELECT lease_expires_at_us FROM background_jobs" + ).fetchone()[0] + ) + # Hold SQLite's writer lock until the heartbeat has provably been + # through its busy timeout more than once AND the lease it was + # claimed under has expired. Waiting on those two facts, rather + # than sleeping a duration chosen to outlast them, is what makes + # the contention this test is about deterministic under load. + deadline = time.monotonic() + 60 + while True: + expired = time.time_ns() // 1_000 > claimed_lease_us + if contended.is_set() and expired: + break + self.assertLess( + time.monotonic(), + deadline, + f"heartbeat never contended (contended={contended.is_set()}, " + f"lease expired={expired})", + ) + time.sleep(0.02) self.connection.commit() runner = ProjectionJobRunner( @@ -406,13 +448,16 @@ def write_locked_build(_workspace_id): clock_us=lambda: time.time_ns() // 1_000, lease_owner="test-worker", token_factory=lambda: "lease-token", - lease_duration_us=100_000, - heartbeat_interval_us=20_000, + lease_duration_us=lease_duration_us, + heartbeat_interval_us=50_000, + heartbeat_connection_factory=heartbeat_connection, retry_delay_us=1, ) result = runner.run_once() + self.assertTrue(contended.is_set()) + self.assertEqual("succeeded", result.status) self.assertEqual( ("succeeded", 1), From db47846a63dc64912e81b7e26b319b385041b9c9 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 05:42:51 -0400 Subject: [PATCH 58/72] =?UTF-8?q?fix:=20round=201=20review=20=E2=80=94=20p?= =?UTF-8?q?rovenance,=20host=20paths,=20and=20the=20writes=20rollback=20hi?= =?UTF-8?q?des?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R-11/R-38 (major): `reconcile_retained_governance` read "absent from the retained v6 rows" as "deleted in v6", and ran on the reactivate path, whose candidate is the database that was live. So apply → author a rule or trigger through v7 → rollback --discard-v7-writes → apply disabled every v7-authored rule and deleted every v7-authored trigger. It now reconciles only streams whose creating governance event was written by the migration itself. R-32 (major, security): `_lossless_row` copied each v6 column as a `[name, value]` pair, so the host-absolute `file_path` rode inside every migrated event's payload and out through `workspace_export` — including from stores migrated by this branch. `_reject_raw_paths` only inspected mapping keys and the wire sweep exempts `payload.data`, so nothing caught it. Host path columns, and a derived relative form that is not workspace-relative, are now written as a `{"$host_path": {sha256,length}}` digest (self-consistent: `_source_row_hash` is only ever compared against itself, and the original stays in `source.snapshot.db` and the retained v6 tables). `_reject_raw_paths` also refuses the pair-list shape, so a regression fails loudly. R-2/R-34/R-13/R-17 (major/minor): the rollback guard counted two of the four tables a v7 client can add user data to. One named `_POST_ACTIVATION_TABLES` now covers `workspace_link_events` (federation) and `memory_capture_candidates` (unapproved user content, plain inserts with no event), and the comparison is against row counts recorded at activation rather than a wall clock, so a backward clock step cannot hide the writes this exists to refuse. R-33/R-36/R-39 (minor): five more readers still refused a migrated row on the legacy `file_path` — local_state, maintenance, discovery, relationship and code_entity — and two passed the stored relative path into `RelativePath` unchecked. All now behave like the readers the plan named. R-5 (minor): the four MCP resources answered "outside its workspace" for a v6 store, which is more misleading than what it replaced; they now raise MIGRATION_REQUIRED like every other entry point. R-12 (minor): the rollback refusal's count and retained candidate never reached the operator, and `migrate-v7` never printed its warnings without --json. Both do now; other migration details stay redacted to their code. R-3 (minor): `RecordSummary.tags` required uniqueness while every tag input allowed duplicates, so an ordinary `memory_store(tags=["a","a"])` wrote a record no reader could render. `UniqueTags` fixes the asymmetry at the input. R-7: a trigger recording no project root is foreign again. R-9/R-15/R-41: removals, edits and unreconcilable trigger edits are counted and warned separately. R-8/R-16: a row that recorded any link shape and kept none is counted. R-14/R-43: one `sole_recorded_project_root`, normalized, for both file links and triggers. R-18/R-40: only the three `memory_*` snapshots skip the live manifest invariants. R-42: a pointerless store mid-creation is not capped, so its recovery branch still heals it. R-35: `_sqlite_backup` opens the source through `_readonly_connection`. R-10: `functools.partial` instead of a parameter two importers never read. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/code_entity_operations.py | 8 +- daem0nmcp/api/v7/discovery_operations.py | 16 +- daem0nmcp/api/v7/local_state_operations.py | 4 +- daem0nmcp/api/v7/maintenance_operations.py | 5 +- daem0nmcp/api/v7/models.py | 6 + daem0nmcp/api/v7/operations.py | 16 ++ daem0nmcp/api/v7/relationship_operations.py | 4 +- daem0nmcp/api/v7/resource_repository.py | 16 ++ daem0nmcp/api/v7/tools.py | 9 +- daem0nmcp/cli.py | 14 +- daem0nmcp/database.py | 20 +- daem0nmcp/migrations/schema.py | 77 +++++- daem0nmcp/migrations/v7.py | 248 +++++++++++++++----- daem0nmcp/verification_v7.py | 14 +- 14 files changed, 364 insertions(+), 93 deletions(-) diff --git a/daem0nmcp/api/v7/code_entity_operations.py b/daem0nmcp/api/v7/code_entity_operations.py index 3d45aed..e9a3daf 100644 --- a/daem0nmcp/api/v7/code_entity_operations.py +++ b/daem0nmcp/api/v7/code_entity_operations.py @@ -49,6 +49,7 @@ from .models import ( EvidenceRef, RecordSummary, + stored_relative_path, ) from .public_ids import ( PublicObjectIdNotFound, @@ -364,7 +365,6 @@ def _validated_record_row( or row["stream_version"] != expected_version or row["state_hash"] != memory_state_hash(state) or row["content_hash"] != memory_content_hash(state) - or row["file_path"] is not None ): raise CodeEntityOperationError("CAPABILITY_DEGRADED") return row @@ -380,17 +380,17 @@ def _event_record_summary( content = state["content"] if not isinstance(content, str) or not content: raise CodeEntityOperationError("CAPABILITY_DEGRADED") + # A record's valid time may precede or follow the transaction time it was + # written at; ``RecordSummary`` shows the earlier of the two. created = _datetime_from_us(occurred_at_us) updated = _datetime_from_us(recorded_at_us) - if created > updated: - raise CodeEntityOperationError("CAPABILITY_DEGRADED") try: return RecordSummary( record_id=record_id, record_type=state["record_type"], excerpt=content[:4000], tags=state["tags"], - relative_file_path=state["file_path_relative"], + relative_file_path=stored_relative_path(state["file_path_relative"]), current_status=( "invalidated" if state["deleted_at_us"] is not None diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 14b7ace..d345d1f 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -50,7 +50,13 @@ ) from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import CapabilityState, Page, RecordSummary, RetrievalData +from .models import ( + CapabilityState, + Page, + RecordSummary, + RetrievalData, + stored_relative_path, +) from .public_ids import ( PublicObjectIdNotFound, PublicObjectIdRepository, @@ -1042,24 +1048,24 @@ def _record_summary(row: sqlite3.Row) -> RecordSummary: try: content = row["content"] tags = json.loads(str(row["tags_json"])) + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote and + # may carry a valid time ahead of its transaction time; neither is + # corruption, and ``RecordSummary`` bounds what is emitted. if ( not isinstance(content, str) or not content or not isinstance(tags, list) - or row["file_path"] is not None or row["deleted_at_us"] is not None ): raise ValueError created = _datetime_from_us(row["created_at_us"]) updated = _datetime_from_us(row["updated_at_us"]) - if updated < created: - raise ValueError return RecordSummary( record_id=row["record_id"], record_type=row["record_type"], excerpt=content[:4000], tags=tags, - relative_file_path=row["file_path_relative"], + relative_file_path=stored_relative_path(row["file_path_relative"]), current_status="archived" if bool(row["archived"]) else "current", content_hash=row["content_hash"], created_at=created, diff --git a/daem0nmcp/api/v7/local_state_operations.py b/daem0nmcp/api/v7/local_state_operations.py index c83969d..2dc7cc4 100644 --- a/daem0nmcp/api/v7/local_state_operations.py +++ b/daem0nmcp/api/v7/local_state_operations.py @@ -278,8 +278,8 @@ def _json_string_list(value: object) -> list[str]: def _record_summary(row: sqlite3.Row) -> RecordSummary: - if row["file_path"] is not None: - raise LocalStateOperationError("CAPABILITY_DEGRADED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it + # is legacy provenance, never emitted, so it must not deny the read. content = row["content"] record_type = row["record_type"] archived = row["archived"] diff --git a/daem0nmcp/api/v7/maintenance_operations.py b/daem0nmcp/api/v7/maintenance_operations.py index 977bc98..fcaa689 100644 --- a/daem0nmcp/api/v7/maintenance_operations.py +++ b/daem0nmcp/api/v7/maintenance_operations.py @@ -305,9 +305,10 @@ def _verified_snapshot(row: sqlite3.Row) -> _RecordSnapshot: try: state = _record_state(row) payload = _parse_json(row["payload_json"], dict) + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; + # ``_record_state`` already drops it from the compared state. if ( - row["file_path"] is not None - or row["workspace_id"] != row["event_workspace_id"] + row["workspace_id"] != row["event_workspace_id"] or row["record_id"] != row["event_stream_id"] or payload.get("record") != state or canonical_json_bytes(payload).decode("utf-8") != str(row["payload_json"]) diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index 55d6d09..85fd236 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -410,6 +410,11 @@ def _document_validation_error( UserText, StringConstraints(strict=True, min_length=1, max_length=80), ] +# Every stored tag list. ``RecordSummary.tags`` requires uniqueness on the +# way out, so the inputs that feed it require it on the way in; without it a +# plain ``memory_store(tags=['a','a'])`` wrote a record no reader could +# render. +UniqueTags = Annotated[list[Tag], AfterValidator(_unique_strings)] ProviderName = Annotated[ str, StringConstraints( @@ -939,6 +944,7 @@ class Preview(WireModel): "RuleId", "SelectionToken", "Tag", + "UniqueTags", "TokenUsage", "ToolName", "TriggerId", diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index a65bc12..8771574 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -505,6 +505,22 @@ def _reject_raw_paths(value: object) -> None: if key != "context": _reject_raw_paths(item) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + # A migrated event carries its v6 row as ``[name, value]`` pairs, so a + # path column is a list element rather than a mapping key. Check that + # shape too: the migration replaces these with a digest, and a + # regression must fail here rather than reach the wire. + if ( + len(value) == 2 + and value[0] in _RAW_PATH_KEYS | _RELATIVE_PATH_KEYS + and isinstance(value[1], str) + and value[1] + ): + if value[0] in _RAW_PATH_KEYS: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") + try: + _RELATIVE_PATH_ADAPTER.validate_python(value[1]) + except ValidationError: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") from None for item in value: _reject_raw_paths(item) elif isinstance(value, (bytes, bytearray, memoryview)): diff --git a/daem0nmcp/api/v7/relationship_operations.py b/daem0nmcp/api/v7/relationship_operations.py index 08ff11b..c6c8b92 100644 --- a/daem0nmcp/api/v7/relationship_operations.py +++ b/daem0nmcp/api/v7/relationship_operations.py @@ -353,7 +353,9 @@ async def _run_read( def _record_summary(row: sqlite3.Row) -> RecordSummary: try: content = row["content"] - if not isinstance(content, str) or not content or row["file_path"] is not None: + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it + # is legacy provenance, never emitted, so it must not deny the read. + if not isinstance(content, str) or not content: raise ValueError tags = json.loads(str(row["tags_json"])) if not isinstance(tags, list): diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index ea1ffb1..c08bd95 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -216,6 +216,15 @@ def __init__(self, code: str = "RESOURCE_REPOSITORY_UNAVAILABLE") -> None: super().__init__(code) +class ResourceMigrationRequiredError(RuntimeError): + """The workspace still uses the v6 storage format.""" + + code = "MIGRATION_REQUIRED" + + def __init__(self) -> None: + super().__init__(self.code) + + ActiveDatabaseResolver = Callable[[Workspace], ResolvedActiveDatabase] @@ -1000,6 +1009,10 @@ def _open_connection( raise TypeError( "active database resolver returned an invalid selection" ) + if selected.format_version == 6: + # Retrying never helps, and "outside its workspace" below would + # be a more misleading answer than naming the real reason. + raise ResourceMigrationRequiredError() if selected.format_version != 7: raise ValueError("active database is not architecture format 7") storage = normalize_resolved_path( @@ -1007,6 +1020,9 @@ def _open_connection( ) database_path = normalize_resolved_path(selected.path.resolve(strict=True)) database_path.relative_to(storage) + except ResourceMigrationRequiredError: + storage_lock.release() + raise except (OSError, RuntimeError, ValueError) as exc: storage_lock.release() raise ValueError( diff --git a/daem0nmcp/api/v7/tools.py b/daem0nmcp/api/v7/tools.py index b1fa7c9..4c48ab9 100644 --- a/daem0nmcp/api/v7/tools.py +++ b/daem0nmcp/api/v7/tools.py @@ -53,6 +53,7 @@ Tag, ToolName, TriggerId, + UniqueTags, UserJsonObject, UserText, UtcDateTime, @@ -1118,7 +1119,7 @@ class MemoryStoreInput(WireModel): | None ) = None context: RecordContext = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=32) + tags: UniqueTags = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) @@ -1154,7 +1155,7 @@ class MemoryCapturePromoteInput(WireModel): ] rationale: OptionalUserMediumText = None context: RecordContext = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=16) + tags: UniqueTags = Field(default_factory=list, max_length=16) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1185,7 +1186,7 @@ class ActiveContextListInput(WireModel): class ContextTriggersMatchInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath | None = None - tags: list[Tag] = Field(default_factory=list, max_length=32) + tags: UniqueTags = Field(default_factory=list, max_length=32) entities: list[UserNameText] = Field(default_factory=list, max_length=32) limit: Annotated[int, Field(ge=1, le=20)] = 5 @@ -1273,7 +1274,7 @@ class MemoryCreate(WireModel): | None ) = None context: RecordContext = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=32) + tags: UniqueTags = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index 938f39d..607773e 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -704,7 +704,15 @@ def main(): "checkpoints": {}, "validation": {}, "warnings": [], - "error": {"code": exc.code, "message": exc.code}, + # Migration details stay redacted to the code, except the + # rollback refusal, whose detail is the operator signal: a + # count and the storage-relative candidate, never a host path. + "error": { + "code": exc.code, + "message": str(exc) + if exc.code == "ROLLBACK_WOULD_HIDE_WRITES" + else exc.code, + }, } exit_code = 2 if exc.code == "UNSAFE_MIGRATION_PATH" else 1 except Exception: @@ -738,8 +746,10 @@ def main(): ) if payload.get("active_generation") is not None: print(f"Generation: {payload['active_generation']}") + for warning in payload.get("warnings") or (): + print(f"Warning: {warning}") if payload.get("error"): - print(f"Error: {payload['error']['code']}") + print(f"Error: {payload['error']['message']}") sys.exit(exit_code) if args.command == "recover-consolidation": diff --git a/daem0nmcp/database.py b/daem0nmcp/database.py index d975e4d..e325b34 100644 --- a/daem0nmcp/database.py +++ b/daem0nmcp/database.py @@ -209,13 +209,21 @@ def _run_migrations( self._migrated = True - def _v6_migration_ceiling(self) -> int | None: - """The migration ceiling for a retained format-6 store, else ``None``.""" + def _v6_migration_ceiling(self, previous_schema_version: int) -> int | None: + """The migration ceiling for a retained format-6 store, else ``None``. + + A brand-new database whose creation crashed part-way through its own + migration run is pointerless and therefore reads as format 6, but it + holds no user rows and is finished by the recovery branch in + ``init_db``; capping it would leave it permanently incomplete. + """ - if self.format_version != 6: - return None from .migrations.schema import _LAST_V6_SCHEMA_VERSION + if self.format_version != 6: + return None + if previous_schema_version >= 16 and not self._has_user_rows(): + return None return _LAST_V6_SCHEMA_VERSION async def init_db(self): @@ -237,7 +245,9 @@ async def init_db(self): # to. Applying the v7 ledger to it in place would rewrite the # source without a backup, and its governance backfill then goes # stale against later v6 edits, so cap it at the last v6 version. - self._run_migrations(maximum_version=self._v6_migration_ceiling()) + self._run_migrations( + maximum_version=self._v6_migration_ceiling(previous_schema_version) + ) # Then create any new tables async with self.engine.begin() as conn: diff --git a/daem0nmcp/migrations/schema.py b/daem0nmcp/migrations/schema.py index 5062e6f..115efd7 100644 --- a/daem0nmcp/migrations/schema.py +++ b/daem0nmcp/migrations/schema.py @@ -2891,8 +2891,9 @@ def _retained_trigger_states( "SELECT id,project_path,trigger_type,pattern,recall_topic,recall_categories," "is_active,priority,created_at FROM context_triggers ORDER BY id" ).fetchall() - recorded = {row[1] for row in rows if isinstance(row[1], str) and row[1]} - sole_root = recorded.pop() if len(recorded) == 1 else None + from daem0nmcp.migrations.v7 import sole_recorded_project_root + + sole_root = sole_recorded_project_root(connection) public_types = { "file_pattern": "file", "tag_match": "tag", @@ -2902,7 +2903,17 @@ def _retained_trigger_states( foreign = 0 for row in rows: source_id = _retained_source_id(row[0]) - if row[1] != sole_root and _retained_path_workspace_id(row[1]) != workspace_id: + # A trigger that records no project root names no workspace, so it + # stays foreign exactly as it was before the sole-root rule existed. + local = ( + isinstance(row[1], str) + and bool(row[1]) + and ( + row[1] == sole_root + or _retained_path_workspace_id(row[1]) == workspace_id + ) + ) + if not local: foreign += 1 continue trigger_type = public_types.get(row[2]) @@ -2940,6 +2951,21 @@ def _retained_trigger_states( return states, foreign +def _migration_authored_streams( + connection: sqlite3.Connection, workspace_id: str +) -> set[str]: + """The governance streams the retained backfill created, not v7 clients.""" + + return { + str(row[0]) + for row in connection.execute( + "SELECT stream_id FROM governance_events WHERE workspace_id=? " + "AND stream_version=1 AND actor_type='migration'", + (workspace_id,), + ) + } + + def _governance_stream_exists( connection: sqlite3.Connection, workspace_id: str, stream_id: str ) -> bool: @@ -3030,9 +3056,17 @@ def reconcile_retained_governance( stream that already exists, so without this pass the edit or the deletion would never reach v7 and a `must_not` could silently disappear. + Only streams the backfill itself created are reconciled. A reactivated + candidate is the database that *was* live, so it also holds rules and + triggers authored through v7, which have no retained v6 row; inferring + "deleted in v6" from their absence would disable or delete them. The + creating event's `actor_type` separates the two: the backfill writes + `migration`, a v7 client writes `client`. + The governance event vocabulary has `rule.updated` and - `context_trigger.deleted`; a rule that no longer exists in v6 is therefore - recorded as disabled rather than removed. + `context_trigger.deleted` but no `context_trigger.updated`, so a rule that + no longer exists in v6 is recorded as disabled, and a trigger whose + content changed in v6 is counted and reported instead of rewritten. """ from daem0nmcp.event_store import ( @@ -3043,7 +3077,13 @@ def reconcile_retained_governance( if not _table_exists(connection, "governance_events"): raise RuntimeError("GOVERNANCE_SCHEMA_INCOMPLETE") store = GovernanceEventStore(connection, assume_transaction=True) - counts = {"rules_updated": 0, "triggers_deleted": 0} + counts = { + "rules_updated": 0, + "rules_disabled": 0, + "triggers_deleted": 0, + "triggers_unreconciled": 0, + } + migrated = _migration_authored_streams(connection, workspace_id) rule_states = _retained_rule_states(connection, workspace_id) for row in connection.execute( @@ -3053,6 +3093,8 @@ def reconcile_retained_governance( (workspace_id,), ).fetchall(): public_id = str(row[0]) + if public_id not in migrated: + continue entry = rule_states.get(public_id) current = { "rule_id": public_id, @@ -3064,7 +3106,8 @@ def reconcile_retained_governance( "priority": row[6], "enabled": bool(row[7]), } - if entry is None: + removed = entry is None + if removed: desired = dict(current, enabled=False) correlation = f"migration21:rule-removed:{public_id}" else: @@ -3094,7 +3137,7 @@ def reconcile_retained_governance( expected_stream_version=int(row[8]) + 1, ) ) - counts["rules_updated"] += 1 + counts["rules_disabled" if removed else "rules_updated"] += 1 trigger_states, _foreign = _retained_trigger_states(connection, workspace_id) for row in connection.execute( @@ -3105,7 +3148,23 @@ def reconcile_retained_governance( (workspace_id,), ).fetchall(): public_id = str(row[0]) - if public_id in trigger_states: + if public_id not in migrated: + continue + entry = trigger_states.get(public_id) + if entry is not None: + # No `context_trigger.updated` event type exists, so a v6 edit (or + # a rowid reused by a different trigger) cannot be replayed. Count + # it so the operator is told rather than left with a stale pattern. + state = entry[1] + if ( + state["trigger_type"] != row[1] + or state["pattern"] != row[2] + or state["recall_query"] != row[3] + or state["categories"] != json.loads(row[4]) + or state["enabled"] != bool(row[5]) + or state["priority"] != row[6] + ): + counts["triggers_unreconciled"] += 1 continue store.append_and_project( GovernanceEventCommand( diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index 2426d3c..3835716 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -20,6 +20,7 @@ from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass, field from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Any from urllib.parse import quote @@ -46,6 +47,7 @@ from ..workspace import ( WorkspaceRegistry, is_workspace_relative_path, + normalize_resolved_path, relative_to_root, resolve_derived_path, ) @@ -56,6 +58,16 @@ TARGET_FORMAT_VERSION = 7 DEFAULT_BATCH_SIZE = 500 +# Every table a v7 client can add user data to after activation. The +# three event ledgers are append-only (schema triggers enforce it), so a +# row count is an exact high-water mark; capture candidates are plain +# inserts holding unapproved user content, so a rise is reported too. +_POST_ACTIVATION_TABLES = ( + "memory_events", + "governance_events", + "workspace_link_events", + "memory_capture_candidates", +) # v6 accepted empty content; the v7 bounded summary needs at least one # character. ``api/v7/models.MIGRATED_EMPTY_CONTENT`` uses the same marker for # rows migrated before this normalization existed. @@ -200,6 +212,13 @@ def _table_names(connection: sqlite3.Connection) -> list[str]: ] +def _has_column(connection: sqlite3.Connection, table: str, column: str) -> bool: + return any( + str(row[1]) == column + for row in connection.execute(f"PRAGMA table_info({_quoted_identifier(table)})") + ) + + def _table_exists(connection: sqlite3.Connection, name: str) -> bool: return ( connection.execute( @@ -626,12 +645,11 @@ def _partial_candidates(root: Path | None) -> list[Path]: def _sqlite_backup(source: Path, destination: Path) -> None: - source_uri = "file:" + quote(source.resolve().as_posix(), safe="/:") + "?mode=ro" - source_connection = sqlite3.connect(source_uri, uri=True) + # Opened through the same reader the dry run uses, so the two do not + # disagree about what "read-only" means beside the v6 file. + source_connection = _readonly_connection(source) target_connection = sqlite3.connect(destination) try: - source_connection.execute("PRAGMA foreign_keys=ON") - source_connection.execute("PRAGMA busy_timeout=0") source_connection.backup(target_connection) target_connection.commit() finally: @@ -655,27 +673,51 @@ def _integrity(path: Path) -> dict[str, Any]: return {"integrity_check": "ok", "foreign_key_violations": foreign} +# v6 columns that hold a path on the machine that wrote them. The lossless +# row rides inside every migrated event's payload, and an export bundle is a +# shareable artifact, so the value is replaced by a digest rather than copied +# to the wire. The original stays in `source.snapshot.db` and in the retained +# v6 tables, and the usable form is derived into `record.file_path_relative`. +_HOST_PATH_COLUMNS = frozenset({"file_path", "project_path", "database_path"}) +_DERIVED_PATH_COLUMNS = frozenset({"file_path_relative", "relative_file_path"}) + + +def _host_path_marker(value: str) -> dict[str, Any]: + return { + "$host_path": { + "length": len(value), + "sha256": hashlib.sha256(value.encode("utf-8")).hexdigest(), + "snapshot": "source.snapshot.db", + } + } + + +def _lossless_column(row: sqlite3.Row, table: str, name: str) -> Any: + value = row[name] + if table == "memories" and name == "vector_embedding" and isinstance(value, bytes): + return { + "$legacy_vector": { + "length": len(value), + "sha256": hashlib.sha256(value).hexdigest(), + "snapshot": "source.snapshot.db", + } + } + if isinstance(value, str) and value: + if name in _HOST_PATH_COLUMNS: + return _host_path_marker(value) + # v6 wrote `../outside/x.py` here for a file above the project, and + # the absolute path itself across Windows drives; neither may leave + # the host either. + if name in _DERIVED_PATH_COLUMNS and not is_workspace_relative_path(value): + return _host_path_marker(value) + return _encode_sqlite_value(value, table=table, column=name) + + def _lossless_row(row: sqlite3.Row, table: str) -> dict[str, Any]: return { "table": table, "columns": [ - [ - name, - ( - { - "$legacy_vector": { - "length": len(row[name]), - "sha256": hashlib.sha256(row[name]).hexdigest(), - "snapshot": "source.snapshot.db", - } - } - if table == "memories" - and name == "vector_embedding" - and isinstance(row[name], bytes) - else _encode_sqlite_value(row[name], table=table, column=name) - ), - ] - for name in tuple(row.keys()) + [name, _lossless_column(row, table, name)] for name in tuple(row.keys()) ], } @@ -731,17 +773,18 @@ def _legacy_score(value: Any) -> float | None: return number if math.isfinite(number) and 0 <= number <= 1 else None -def _legacy_project_roots( - connection: sqlite3.Connection, workspace_root: Path -) -> tuple[str, ...]: - """The roots a v6 file link may be relative to, workspace root first. +def sole_recorded_project_root(connection: sqlite3.Connection) -> str | None: + """The one project root a v6 database records, or ``None``. - A v6 database records the project root it was written for. When it holds - exactly one and it is not this workspace, the project was moved before the - migration, so file links are resolved against it as well. + v6 wrote the project root onto rows in several tables. When every one of + them names the same root, that root is this database's own project even + after the directory was moved; when they disagree, no single root can be + inferred. Spellings are compared normalized, so a trailing separator or + Windows case does not split one root into two. This is the single + definition: file links and context triggers both use it. """ - recorded: set[str] = set() + recorded: dict[str, str] = {} for table in _table_names(connection): if table in _V7_TABLE_NAMES: continue @@ -753,16 +796,44 @@ def _legacy_project_roots( } if "project_path" not in columns: continue - recorded.update( - value - for value in _distinct(connection, table, "project_path") - if isinstance(value, str) and value - ) + for value in _distinct(connection, table, "project_path"): + if not isinstance(value, str) or not value: + continue + recorded.setdefault(_project_root_key(value), value) + if len(recorded) > 1: + return None + if len(recorded) != 1: + return None + return next(iter(recorded.values())) + + +def _project_root_key(value: str) -> str: + try: + resolved = normalize_resolved_path(Path(value).expanduser().resolve()) + except (OSError, RuntimeError, ValueError): + return os.path.normcase(value.rstrip("/\\")) + return os.path.normcase(str(resolved)) + + +def _legacy_project_roots( + connection: sqlite3.Connection, workspace_root: Path +) -> tuple[str, ...]: + """The roots a v6 file link may be relative to, workspace root first. + + When the database records exactly one project root and it is not this + workspace, the project was moved before the migration, so file links are + resolved against it as well. Containment is decided by + ``relative_to_root``, which compares ``os.path.normcase`` forms; on a + case-insensitive volume whose real spelling differs from the recorded one + the link is dropped and counted rather than guessed. + """ + roots = [str(workspace_root)] - if len(recorded) == 1: - recorded_root = next(iter(recorded)) - if os.path.normcase(recorded_root) != os.path.normcase(roots[0]): - roots.append(recorded_root) + recorded_root = sole_recorded_project_root(connection) + if recorded_root is not None and _project_root_key( + recorded_root + ) != _project_root_key(roots[0]): + roots.append(recorded_root) return tuple(roots) @@ -1124,12 +1195,24 @@ def _migration_warnings(validation: dict[str, Any]) -> tuple[str, ...]: f"{updated} rule(s) changed in v6 after an earlier in-place upgrade and " "were reconciled" ) + disabled = int(validation.get("rules_disabled", 0)) + if disabled: + warnings.append( + f"{disabled} rule(s) removed in v6 after an earlier in-place upgrade " + "were disabled; no rule deletion event exists" + ) deleted = int(validation.get("triggers_deleted", 0)) if deleted: warnings.append( f"{deleted} context trigger(s) removed in v6 after an earlier in-place " "upgrade were marked deleted" ) + unreconciled = int(validation.get("triggers_unreconciled", 0)) + if unreconciled: + warnings.append( + f"{unreconciled} context trigger(s) were edited in v6 after an earlier " + "in-place upgrade and keep the earlier content; re-create them" + ) return tuple(warnings) @@ -1340,6 +1423,22 @@ def _mark_candidate_active( raise MigrationV7Error( "ACTIVATION_STATE_INVALID", "candidate run is not ready" ) + validation_row = active.execute( + "SELECT validation_json FROM v7_migration_runs " + "WHERE migration_run_id=?", + (run_id,), + ).fetchone() + try: + validation = json.loads(validation_row[0] or "{}") + except (TypeError, ValueError): + validation = {} + if isinstance(validation, dict): + validation["activation_counts"] = self._post_activation_counts(active) + active.execute( + "UPDATE v7_migration_runs SET validation_json=? " + "WHERE migration_run_id=?", + (canonical_json_bytes(validation).decode("utf-8"), run_id), + ) active.execute( "UPDATE projection_manifests SET status='active', activated_at_us=? " "WHERE workspace_id=? AND status='ready' AND generation=1 " @@ -2024,7 +2123,7 @@ def _import_candidate( connection.execute("PRAGMA synchronous=FULL") roots = _legacy_project_roots(connection, workspace_root) for table, importer in ( - ("memories", self._import_memory_row), + ("memories", partial(self._import_memory_row, roots=roots)), ("facts", self._import_fact_row), ("memory_relationships", self._import_relationship_row), ): @@ -2061,9 +2160,7 @@ def _import_candidate( break store = EventStore(connection) for row in rows: - importer( - connection, store, row, run_id, workspace_id, roots - ) + importer(connection, store, row, run_id, workspace_id) row_hash = _source_row_hash(row, table) rolling = sha256_json([rolling, row_hash]) last_pk = int(row["id"]) @@ -2105,6 +2202,7 @@ def _import_memory_row( row: sqlite3.Row, run_id: str, workspace_id: str, + *, roots: tuple[str, ...], ) -> None: legacy_id = str(row["id"]) @@ -2297,7 +2395,6 @@ def _import_fact_row( row: sqlite3.Row, run_id: str, workspace_id: str, - roots: tuple[str, ...], ) -> None: legacy_id = str(row["id"]) fact_id = deterministic_id( @@ -2383,7 +2480,6 @@ def _import_relationship_row( row: sqlite3.Row, run_id: str, workspace_id: str, - roots: tuple[str, ...], ) -> None: occurred, quality, original = _parse_legacy_time(_column(row, "created_at")) source = self._mapped_memory( @@ -2485,6 +2581,9 @@ def _finalize_candidate( # A legacy command may already have backfilled governance into the # v6 file; the v6 tables can have changed since. Carry those edits # over before the candidate is declared ready. + # `_run_migration_16` already ran the ledger's own + # backfill on this candidate, so every retained rule and trigger + # has a public id and a stream before the comparison below. governance = reconcile_retained_governance( connection, workspace_id, now_us=now ) @@ -3092,15 +3191,20 @@ def _finalize_candidate( now, ), ) + # Count every head row that recorded a file link of any shape and + # kept none. Versions inherit the head's link, so they cannot lose + # one independently. dropped_file_links = 0 if _table_exists(connection, "memories"): for source_row in connection.execute("SELECT * FROM memories"): - absolute = _column(source_row, "file_path") - if ( - isinstance(absolute, str) - and absolute - and _legacy_relative_path(source_row, roots) is None - ): + recorded = any( + isinstance(value, str) and value + for value in ( + _column(source_row, "file_path"), + _column(source_row, "file_path_relative"), + ) + ) + if recorded and _legacy_relative_path(source_row, roots) is None: dropped_file_links += 1 validation = { "integrity_check": "ok", @@ -3115,7 +3219,9 @@ def _finalize_candidate( "rule_count": _count(connection, "governance_rules"), "trigger_count": _count(connection, "governance_context_triggers"), "rules_updated": governance["rules_updated"], + "rules_disabled": governance["rules_disabled"], "triggers_deleted": governance["triggers_deleted"], + "triggers_unreconciled": governance["triggers_unreconciled"], "foreign_trigger_count": foreign_triggers, "dropped_file_link_count": dropped_file_links, } @@ -3164,26 +3270,60 @@ def _restore_pointer(self, storage: Path, previous) -> None: return write_active_pointer(storage, previous.pointer) + @staticmethod + def _post_activation_counts(connection: sqlite3.Connection) -> dict[str, int]: + """Row counts for every table a v7 client can add user data to.""" + + counts: dict[str, int] = {} + for table in _POST_ACTIVATION_TABLES: + if not _table_exists(connection, table): + continue + counts[table] = int( + connection.execute( + f"SELECT count(*) FROM {_quoted_identifier(table)}" + ).fetchone()[0] + ) + return counts + def _events_after_activation(self, candidate: Path, run_id: str) -> int: - """Count canonical events written to the candidate since activation. + """Count user writes made to the candidate since activation. Rolling the pointer back to v6 makes them unreachable: the v6 file never received them, and a later v6 write changes the run identity, so the next `--apply` builds a different candidate. + + The comparison is against the row counts recorded when the candidate + was activated, not against a timestamp: a backward clock step (NTP, + a resumed VM) would otherwise hide exactly the writes this refuses to + discard. A store activated before this was recorded falls back to the + activation timestamp. """ connection = sqlite3.connect(candidate) try: row = connection.execute( - "SELECT activated_at_us FROM v7_migration_runs WHERE migration_run_id=?", + "SELECT activated_at_us,validation_json FROM v7_migration_runs " + "WHERE migration_run_id=?", (run_id,), ).fetchone() if row is None or row[0] is None: return 0 activated = int(row[0]) + try: + recorded = json.loads(row[1] or "{}").get("activation_counts") + except (TypeError, ValueError): + recorded = None + current = self._post_activation_counts(connection) + if isinstance(recorded, dict): + return sum( + max(0, count - int(recorded.get(table, 0))) + for table, count in current.items() + ) total = 0 - for table in ("memory_events", "governance_events"): - if not _table_exists(connection, table): + for table in _POST_ACTIVATION_TABLES: + if table not in current or not _has_column( + connection, table, "recorded_at_us" + ): continue total += int( connection.execute( diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index f1553c8..ea8c56a 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -78,17 +78,19 @@ "workspace_link_events", } ) -# The snapshots `migrate-v7` records for the tables it fills directly; no -# retrieval provider serves them. +# The snapshots `migrate-v7` activates for the tables it fills directly; no +# retrieval provider serves them, so nothing rebuilds them and the live +# manifest invariants cannot hold once anything is written. The migration's +# `entities`/`communities` manifests are written `rebuild_required`, so they +# keep their own row-count verification. _TABLE_SNAPSHOT_PROJECTIONS = frozenset( { "memory_records", "memory_fact_versions", "memory_relationship_versions", - "entities", - "communities", } ) +_DISCOVERY_PROJECTIONS = frozenset({"entities", "communities"}) _LOCAL_PROJECTIONS = frozenset( { "memory_records", @@ -101,7 +103,9 @@ "outcome", } ) -_SUPPORTED_PROJECTIONS = _TABLE_SNAPSHOT_PROJECTIONS | RETRIEVAL_PROJECTION_NAMES +_SUPPORTED_PROJECTIONS = ( + _TABLE_SNAPSHOT_PROJECTIONS | _DISCOVERY_PROJECTIONS | RETRIEVAL_PROJECTION_NAMES +) class VerificationV7Error(RuntimeError): From e288e239cd589ede4646ea42c2b9c682250483e4 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 06:05:55 -0400 Subject: [PATCH 59/72] test: pin the repairs the review found untested, and clip empty content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R-22 (major): the "already-migrated store" test looped over `recall["items"]`, which was empty, so both reader-side repairs were unproved — neutering `stored_relative_path` or deleting the bound clipping left it green. It now reads the affected record through `memory_search_text` with metadata (so the assertions run on values the test controls), over each shape v6 could store: `../outside`, the drive-lettered `resolved.as_posix()` fallback that `_v6_file_paths` was missing, and a POSIX absolute. It asserts no link is emitted, the host path is absent from the whole response, the tags are 32 unique names of ≤80 characters, and the excerpt of an empty v6 content is the migration marker. `test_the_bounded_summary_clips_what_v6_had_no_limit_for` pins the model itself. That test found a real defect: `RecordSummary` substitutes the marker, but four readers refused an empty stored content *before* building one, so a single empty migrated record still failed session_brief for the whole workspace — the exact failure mode PR 5 escalated. Fixed in resource_repository, record_operations (summary and search excerpt) and runtime_services. R-21 (major): `verify_v7` is asserted `verified` right after a migration and again after the first write through the server, with the table-snapshot manifests asserted present so it cannot pass vacuously. R-11/R-38: a reactivate test that creates a rule and a trigger through v7, removes the retained v6 rows their compatibility write made, rolls back with --discard-v7-writes and re-applies. Measured, and reported in the summary: the reviewers' sequence does not reproduce as written, because `rule_create` and `context_trigger_create` also write the retained v6 row, so the stream is found and left alone. The guard is still right, and this pins it. R-2/R-13/R-34: a federation link event and a pending capture candidate each make rollback refuse. R-28: the refusal and the discard warning are asserted for the count, the candidate and the flag, not for an absent substring. R-26: MIGRATION_REQUIRED over brief, recall, store, the resource surface and the transfer boundary (reached by rolling a briefed session's store back to v6). R-6: the post-repair `content_hash` move is pinned as a decision. R-24/R-25: the fixture builds all sixteen v6 ORM tables and its DDL digest is pinned — it is now byte-identical to the database the v6 code produced, 23 tables and the same sha256. R-20: a store a legacy command touched verifies after migrating. R-29 `entity_match`, R-31 the control-character tag, R-27 the single-value parametrize, R-30 `__import__`. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/models.py | 1 + daem0nmcp/api/v7/record_operations.py | 8 +- daem0nmcp/api/v7/resource_repository.py | 11 +- daem0nmcp/api/v7/runtime_services.py | 4 +- tests/test_migrate_v7_end_to_end.py | 542 ++++++++++++++++++++++-- 5 files changed, 518 insertions(+), 48 deletions(-) diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index 85fd236..b12a635 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -914,6 +914,7 @@ class Preview(WireModel): "FieldError", "guarded_strings", "is_host_absolute_path", + "MIGRATED_EMPTY_CONTENT", "stored_relative_path", "JsonObject", "JsonValue", diff --git a/daem0nmcp/api/v7/record_operations.py b/daem0nmcp/api/v7/record_operations.py index 1bdb3b4..2dccc82 100644 --- a/daem0nmcp/api/v7/record_operations.py +++ b/daem0nmcp/api/v7/record_operations.py @@ -46,6 +46,7 @@ from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET from .models import ( + MIGRATED_EMPTY_CONTENT, EvidenceRef, MutationReceipt, Page, @@ -329,8 +330,9 @@ def _summary( include_metadata: bool = True, ) -> RecordSummary: content = row["content"] - if not isinstance(content, str) or not content: + if not isinstance(content, str): raise RecordOperationError("CAPABILITY_DEGRADED") + content = content or MIGRATED_EMPTY_CONTENT tags = _parse_json(row["tags_json"], list) # A future valid-time may be recorded now; ``RecordSummary`` exposes the # transaction-time creation while the event retains the exact valid time. @@ -1071,7 +1073,9 @@ def _text_search_sync( or bool(row["archived"]) ): raise RecordOperationError("LEXICAL_UNAVAILABLE") - excerpt = str(row["content"])[:8000] + # v6 accepted empty content; the bounded excerpt shows + # the migration marker rather than failing the search. + excerpt = str(row["content"])[:8000] or MIGRATED_EMPTY_CONTENT evidence = candidate.evidence hits.append( TextSearchHit( diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index c08bd95..7fb7cf3 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -32,7 +32,11 @@ from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path from .errors import is_database_busy -from .models import RecordSummary, stored_relative_path +from .models import ( + MIGRATED_EMPTY_CONTENT, + RecordSummary, + stored_relative_path, +) from .public_ids import PublicObjectIdRepository from .resources import ( RESOURCE_FETCH_LIMIT, @@ -1133,8 +1137,11 @@ def _record_summary( if not isinstance(record_type, str) or record_type not in _PUBLIC_RECORD_TYPES: raise ValueError("record type is not public") content = row["content"] - if not isinstance(content, str) or not content: + if not isinstance(content, str): raise ValueError("record content is invalid") + # v6 accepted empty content; ``RecordSummary`` renders it as the + # migration's own marker rather than denying the whole read. + content = content or MIGRATED_EMPTY_CONTENT tags = _string_list(row["tags_json"]) archived = _flag(row["archived"]) deleted = row["deleted_at_us"] is not None diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index e83fc54..c2b6e7c 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -59,6 +59,7 @@ validate_directional_links, ) from .models import ( + MIGRATED_EMPTY_CONTENT, CapabilityState, CitationManifestEntry, RecordSummary, @@ -465,8 +466,9 @@ def _record_summary( ) -> RecordSummary: tags = _parse_json(row["tags_json"], list, "MEMORY_RECORD_INTEGRITY_FAILED") content = row["content"] - if not isinstance(content, str) or not content: + if not isinstance(content, str): raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") + content = content or MIGRATED_EMPTY_CONTENT try: return RecordSummary.model_validate( { diff --git a/tests/test_migrate_v7_end_to_end.py b/tests/test_migrate_v7_end_to_end.py index df97c13..0c56897 100644 --- a/tests/test_migrate_v7_end_to_end.py +++ b/tests/test_migrate_v7_end_to_end.py @@ -4,14 +4,20 @@ tables plus the real migration ledger capped at ``_LAST_V6_SCHEMA_VERSION`` -- and seeded with rows shaped exactly as v6 ``remember`` wrote them, including the host-absolute ``file_path``, the ``../outside`` relative form v6 writes for -a file above the project, unbounded tags and empty content. The column -layout was compared against a database produced by running the v6 code at -commit 00809c6; it matches column for column. +a file above the project, unbounded tags and empty content. + +Its DDL is byte-identical to a database produced by running the v6 code at +commit 00809c6: the same twenty-three tables, and the same +``V6_SCHEMA_SHA256`` over their ``sqlite_master.sql``, which +``test_v6_fixture_uses_the_real_migration_ledger`` enforces so a v7-side edit +to a retained v6 model cannot quietly change what "a v6 database" means here. """ from __future__ import annotations +import hashlib import json +import os import shutil import sqlite3 import sys @@ -25,7 +31,11 @@ from daem0nmcp import models from daem0nmcp.api.v7.production import create_v7_server from daem0nmcp.config import Settings -from daem0nmcp.migrations.schema import _LAST_V6_SCHEMA_VERSION, run_migrations +from daem0nmcp.migrations.schema import ( + _LAST_V6_SCHEMA_VERSION, + MIGRATIONS, + run_migrations, +) from daem0nmcp.migrations.v7 import MigrationV7Error, MigrationV7Service from daem0nmcp.storage_activation import resolve_active_database from daem0nmcp.workspace import ( @@ -35,6 +45,9 @@ ) from tests.api_v7.process_client import call, process_client +# Every table v6's own ``init_db`` created, so the migration's source +# inventory, its logical hash and the rollback byte-identity check run over +# the table set a real v6 store has. The first seven carry rows. V6_TABLES = ( "memories", "memory_versions", @@ -43,13 +56,31 @@ "rules", "context_triggers", "active_context", + "code_entities", + "enforcement_bypass_log", + "extracted_entities", + "file_hashes", + "memory_code_refs", + "memory_communities", + "memory_entity_refs", + "project_links", + "session_state", ) +# The v6 column layout, pinned so a v7-side edit to a retained model cannot +# quietly change what "a v6 database" means here. Update it only when the v6 +# format genuinely changes, and say so in the commit. +V6_SCHEMA_SHA256 = "07c197f506e5ba368e3ee71ef132f7bd636872fc9d89432262a20292e46b8b82" PROFILE_ENVIRONMENT = {"DAEM0NMCP_PROFILE": "core"} TIME = "2026-01-01 00:00:00" def _v6_file_paths(file_path: Path, project_path: Path) -> tuple[str, str]: - """Reproduce v6 ``_normalize_file_path`` (v6 memory.py:63-106) exactly.""" + """Reproduce v6 ``_normalize_file_path`` (v6 memory.py:63-106) exactly. + + Including its inner fallback: across Windows drives ``os.path.relpath`` + raises too, and v6 then stored the resolved *absolute* path in the + ``file_path_relative`` column. + """ resolved = file_path.resolve() absolute = str(resolved) @@ -57,9 +88,10 @@ def _v6_file_paths(file_path: Path, project_path: Path) -> tuple[str, str]: try: relative = resolved.relative_to(root).as_posix() except ValueError: - import os.path - - relative = os.path.relpath(resolved, start=root).replace("\\", "/") + try: + relative = os.path.relpath(resolved, start=root).replace("\\", "/") + except ValueError: + relative = resolved.as_posix() if sys.platform == "win32": absolute = absolute.lower() relative = relative.lower() @@ -192,6 +224,14 @@ def _v6_database(root: Path, *, project_path: Path | None = None) -> Path: "'[\"decision\"]',1,3,?,0,NULL)", (str(project), TIME), ) + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(2,?,'entity_match','Postgres.*','storage entities'," + "'[\"decision\"]',1,1,?,0,NULL)", + (str(project), TIME), + ) connection.execute( "INSERT INTO active_context (id,project_path,memory_id,priority,reason," "added_at,expires_at) VALUES (1,?,1,4,'migration focus',?,NULL)", @@ -203,6 +243,23 @@ def _v6_database(root: Path, *, project_path: Path | None = None) -> Path: return storage +def _v6_schema_digest(database: Path) -> str: + """A digest of the v6 tables' DDL, in a stable order.""" + + connection = sqlite3.connect(database) + try: + statements = [ + str(row[0]) + for row in connection.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL " + "ORDER BY name" + ) + ] + finally: + connection.close() + return hashlib.sha256("\n".join(statements).encode("utf-8")).hexdigest() + + def _registry(root: Path) -> WorkspaceRegistry: return WorkspaceRegistry([root], default_root=root) @@ -238,7 +295,10 @@ async def _invoke(client: Client, workspace_id: str, tool: str, **arguments: Any result = await client.call_tool( tool, {"workspace_id": workspace_id, **arguments}, raise_on_error=False ) - assert result.structured_content is not None, tool + if result.structured_content is None: + # A schema rejection by FastMCP never reaches the v7 router. + assert result.is_error, tool + return {"ok": False, "error": {"code": "SCHEMA_REJECTED"}} return result.structured_content @@ -248,6 +308,28 @@ async def _succeed(client: Client, workspace_id: str, tool: str, **arguments: An return response["data"] +async def _protected( + client: Client, workspace_id: str, tool: str, arguments: dict[str, Any] +): + """Call a covenant-protected tool with the preflight token it requires.""" + + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool=tool, + target_arguments=arguments, + description=f"review before {tool}", + ) + return await _succeed( + client, + workspace_id, + tool, + preflight_token=preflight["preflight_token"], + **arguments, + ) + + def test_v6_fixture_uses_the_real_migration_ledger(tmp_path): """The fixture is a chain-built v6 store, not a hand-written table set.""" @@ -262,16 +344,14 @@ def test_v6_fixture_uses_the_real_migration_ledger(tmp_path): connection.close() assert max(versions) == _LAST_V6_SCHEMA_VERSION assert set(versions) >= { - version - for version, _, _ in __import__( - "daem0nmcp.migrations.schema", fromlist=["MIGRATIONS"] - ).MIGRATIONS - if version <= _LAST_V6_SCHEMA_VERSION + version for version, _, _ in MIGRATIONS if version <= _LAST_V6_SCHEMA_VERSION } + # R-25: the layout itself, so a v7-side edit to a retained v6 model cannot + # change what this fixture means without failing here. + assert _v6_schema_digest(storage / "daem0nmcp.db") == V6_SCHEMA_SHA256 -@pytest.mark.parametrize("transport", ["stdio"]) -async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path, transport): +async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path): """F-003/F-004/F-007/F-008/F-053: migrate, brief, recall, outcome, export.""" root = tmp_path / "project" @@ -282,7 +362,8 @@ async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path, transport): result = MigrationV7Service(_registry(root)).apply(root) assert result.status == "activated", result assert result.validation["rule_count"] == 1 - assert result.validation["trigger_count"] == 1 + # Both v6 trigger types the fixture seeds, including ``entity_match``. + assert result.validation["trigger_count"] == 2 # The one memory whose file lives above the project cannot keep a # workspace-relative link, and the migration says so instead of writing a # path that later fails every read. @@ -303,10 +384,12 @@ async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path, transport): tags = json.loads(rows["Migrated rows must stay readable"]["tags_json"]) assert len(tags) == 32 and len(set(tags)) == 32 assert all(len(tag) <= 80 for tag in tags) + # PR 5 removed the control-character refusal; a migrated tag keeps its. + assert "ctrl\x01char" in tags assert "" in rows, sorted(rows) workspace_id = _workspace_id(root) - async with process_client(root, transport) as session: + async with process_client(root, "stdio") as session: brief = await call(session, "session_brief", {"workspace_id": workspace_id}) assert brief["ok"], brief["error"] assert brief["data"]["applicable_rules"], "the migrated v6 rule must brief" @@ -384,8 +467,27 @@ async def test_every_migrated_row_reads_back_through_the_server(tmp_path): await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") -async def test_a_database_migrated_by_the_pre_fix_code_still_reads(tmp_path): - """The values the previous migration wrote must not brick a read.""" +@pytest.mark.parametrize( + "stored_relative", + [ + # v6 wrote this for a file above the project root... + "../other/helper.py", + # ...and this when `os.path.relpath` raised across Windows drives. + "c:/other/helper.py", + # A POSIX absolute path reaches the same column the same way. + "/var/other/helper.py", + ], +) +async def test_a_database_migrated_by_the_pre_fix_code_still_reads( + tmp_path, stored_relative +): + """PR 5 carry-over: the in-place repair, on values this test controls. + + Neither the ``file_path_relative`` normalization nor the tag/content + clipping is reachable through a store this branch migrated -- the + migration already clips and nulls -- so both are exercised here by + writing the pre-fix projection back and reading it through the server. + """ root = tmp_path / "project" (root / "src").mkdir(parents=True) @@ -394,21 +496,20 @@ async def test_a_database_migrated_by_the_pre_fix_code_still_reads(tmp_path): MigrationV7Service(_registry(root)).apply(root) workspace_id = _workspace_id(root) - # Put the projection back into the shape the pre-fix migration left: the - # host-absolute file_path, v6's ``../`` relative form, v6's unbounded tags - # and empty content. No re-migration is offered for such a store. - absolute, relative = _v6_file_paths(tmp_path / "other" / "helper.py", root) + absolute, _relative = _v6_file_paths(tmp_path / "other" / "helper.py", root) + marker = "spandrel" + overlong = "y" * 120 + tags = json.dumps([overlong, "same", "same", *[f"u{n}" for n in range(40)]]) connection = sqlite3.connect(resolve_active_database(storage).path) try: connection.execute( "UPDATE memory_records SET file_path=?,file_path_relative=?,tags_json=?," "content=? WHERE record_type='pattern'", - ( - absolute, - relative, - json.dumps(["y" * 120, "same", "same", *[f"u{n}" for n in range(40)]]), - "", - ), + (absolute, stored_relative, tags, f"The {marker} helper lives outside"), + ) + # An empty v6 content reaches the reader the same way. + connection.execute( + "UPDATE memory_records SET content='' WHERE record_type='decision'" ) connection.commit() finally: @@ -417,12 +518,241 @@ async def test_a_database_migrated_by_the_pre_fix_code_still_reads(tmp_path): async with Client(_server(root)) as client: brief = await _succeed(client, workspace_id, "session_brief") assert isinstance(brief["warnings"], list) - recall = await _succeed(client, workspace_id, "memory_recall", query="helper") - for item in recall["items"]: - assert item["record"]["relative_file_path"] != relative for kind in ("warnings", "failures", "rules", "active-context"): await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") + hits = await _succeed( + client, + workspace_id, + "memory_search_text", + query=marker, + include_metadata=True, + ) + assert hits["items"], hits + record = hits["items"][0]["record"] + # The stored value is not workspace-relative, so no link is emitted -- + # and above all the host path never reaches the wire. + assert record["relative_file_path"] is None + assert absolute not in json.dumps(hits) + assert stored_relative not in json.dumps(hits) + # v6 had no tag length, uniqueness or count limit; the wire has all three. + assert len(record["tags"]) == 32 + assert len(set(record["tags"])) == 32 + assert all(len(tag) <= 80 for tag in record["tags"]) + assert overlong[:80] in record["tags"] + + empty = await _succeed( + client, + workspace_id, + "memory_search_text", + query="postgres", + include_metadata=True, + ) + assert empty["items"], empty + assert empty["items"][0]["record"]["excerpt"] == "" + + +async def test_a_pre_fix_migrated_record_still_accepts_an_outcome(tmp_path): + """R-6: the repair moves `content_hash`, and that is a decision.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + database = resolve_active_database(storage).path + connection = sqlite3.connect(database) + try: + connection.execute( + "UPDATE memory_records SET file_path_relative='../other/helper.py' " + "WHERE record_type='decision'" + ) + connection.commit() + before = connection.execute( + "SELECT record_id,content_hash FROM memory_records " + "WHERE record_type='decision'" + ).fetchone() + finally: + connection.close() + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + outcome = await _succeed( + client, + workspace_id, + "memory_record_outcome", + record_id=before[0], + outcome_text="still true after the repair", + worked=True, + idempotency_key="pre-fix-outcome-0001", + ) + assert outcome["record_id"] == before[0] + assert outcome["worked"] is True + + connection = sqlite3.connect(database) + try: + after = connection.execute( + "SELECT content_hash,file_path_relative FROM memory_records " + "WHERE record_id=?", + (before[0],), + ).fetchone() + finally: + connection.close() + # Dropping the escaping link is part of the record state, so the next + # event's content hash moves although no content changed. Pinned here so + # a client keying on `content_hash` is a known consequence, not a surprise. + assert after[1] is None + assert after[0] != before[1] + + +async def test_verify_v7_stays_valid_across_the_first_write(tmp_path): + """Plan item 8: a just-activated migration verifies, and keeps verifying. + + The migration activates manifests for tables it fills directly and + nothing rebuilds them, so before the exemption the first write moved + every migrated store to `invalid` for good. + """ + + from daem0nmcp.verification_v7 import verify_v7 + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + # Not vacuous: the migration really did record table-snapshot manifests. + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + activated = { + str(row[0]) + for row in connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active'", + (workspace_id,), + ) + } + finally: + connection.close() + assert { + "memory_records", + "memory_fact_versions", + "memory_relationship_versions", + } <= activated + + assert verify_v7(storage, workspace_id)["status"] == "verified" + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written after the migration", + "idempotency_key": "verify-after-write-0001", + }, + description="write after migrating", + ) + await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written after the migration", + idempotency_key="verify-after-write-0001", + preflight_token=preflight["preflight_token"], + ) + + assert verify_v7(storage, workspace_id)["status"] == "verified" + + +async def test_reactivating_keeps_governance_authored_through_v7(tmp_path): + """Reconcile must not read "not in v6" as "deleted in v6". + + `rule_create` and `context_trigger_create` also write the retained v6 + row, so a stream with no v6 row only appears once that compatibility + write is gone -- this test removes it to pin the guard that makes + reconcile speak only for what the migration itself created. + """ + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + rule = await _protected( + client, + workspace_id, + "rule_create", + { + "trigger": "before touching the cache", + "must_do": ["flush it"], + "idempotency_key": "v7-rule-0001", + }, + ) + trigger = await _protected( + client, + workspace_id, + "context_trigger_create", + { + "trigger_type": "tag", + "pattern": "cache.*", + "recall_query": "cache guidance", + "idempotency_key": "v7-trigger-0001", + }, + ) + rule_id = rule["rule_id"] + trigger_id = trigger["trigger_id"] + + # Drop the retained v6 rows the compatibility write made, leaving two + # streams that exist only in v7 -- the shape reconcile must not touch. + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + connection.execute("DELETE FROM rules WHERE id>1") + connection.execute("DELETE FROM context_triggers WHERE id>2") + connection.commit() + finally: + connection.close() + + # The documented recovery path: roll back accepting the stranded writes, + # then re-apply, which reactivates the retained candidate. + service.rollback(root, discard_v7_writes=True) + result = service.apply(root) + assert result.action == "reactivate", result + assert result.validation["rules_disabled"] == 0, result.validation + assert result.validation["triggers_deleted"] == 0, result.validation + + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + rule_row = connection.execute( + "SELECT enabled FROM governance_rules WHERE rule_id=?", (rule_id,) + ).fetchone() + trigger_row = connection.execute( + "SELECT deleted_at_us FROM governance_context_triggers WHERE trigger_id=?", + (trigger_id,), + ).fetchone() + migrated_rule = connection.execute( + "SELECT enabled FROM governance_rules WHERE rule_id<>?", (rule_id,) + ).fetchone() + finally: + connection.close() + assert rule_row is not None and rule_row[0] == 1, "a v7 rule was disabled" + assert trigger_row is not None and trigger_row[0] is None, ( + "a v7 trigger was deleted" + ) + assert migrated_rule is not None and migrated_rule[0] == 1 + async def test_a_moved_project_keeps_its_file_links(tmp_path): """F-022: v6 recorded one project root; file links resolve against it.""" @@ -440,7 +770,7 @@ async def test_a_moved_project_keeps_its_file_links(tmp_path): result = MigrationV7Service(_registry(new_root)).apply(new_root) assert result.status == "activated", result # The single recorded v6 project path is this workspace's own history. - assert result.validation["trigger_count"] == 1 + assert result.validation["trigger_count"] == 2 assert result.validation["foreign_trigger_count"] == 0 rows = {row["record_type"]: row for row in _records(storage)} assert rows["decision"]["file_path_relative"] == "src/db.py" @@ -451,6 +781,33 @@ async def test_a_moved_project_keeps_its_file_links(tmp_path): await _succeed(client, workspace_id, "memory_recall", query="postgres") +def test_the_bounded_summary_clips_what_v6_had_no_limit_for(): + """The model is the one place every stored-record reader passes through.""" + + from daem0nmcp.api.v7.models import RecordSummary + + base = { + "record_id": "mem_" + "a" * 64, + "record_type": "decision", + "tags": ["x" * 120, "dup", "dup", *[f"t{n}" for n in range(40)]], + "relative_file_path": None, + "current_status": "current", + "content_hash": "b" * 64, + "created_at": "2026-01-02T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + summary = RecordSummary.model_validate({**base, "excerpt": ""}) + # v6 accepted empty content; the wire needs at least one character. + assert summary.excerpt == "" + # v6 tags had no length, uniqueness or count limit. + assert len(summary.tags) == 32 + assert len(set(summary.tags)) == 32 + assert all(len(tag) <= 80 for tag in summary.tags) + # A valid time ahead of the transaction time is shown as the earlier one, + # not refused: v6 kept the two independently. + assert summary.created_at == summary.updated_at + + def test_a_sibling_directory_is_not_inside_the_workspace(tmp_path): """R-25: containment is decided by path parts, never a string prefix.""" @@ -507,15 +864,71 @@ async def test_rollback_refuses_to_hide_writes_made_since_activation(tmp_path): with pytest.raises(MigrationV7Error) as refusal: service.rollback(root) assert refusal.value.code == "ROLLBACK_WOULD_HIDE_WRITES" + # The operator is told how much would be lost and where it stays. + assert "1 event(s)" in str(refusal.value) + assert "candidate.db" in str(refusal.value) + assert "--discard-v7-writes" in str(refusal.value) assert resolve_active_database(storage).format_version == 7 result = service.rollback(root, discard_v7_writes=True) assert result.status == "rolled_back" - assert result.warnings and "unreachable" not in result.warnings[0] + assert any( + "1 event(s)" in warning and "candidate.db" in warning + for warning in result.warnings + ), result.warnings assert resolve_active_database(storage).format_version == 6 assert (storage / "daem0nmcp.db").read_bytes() == source_bytes +@pytest.mark.parametrize( + "table,insert", + [ + ( + "workspace_link_events", + "INSERT INTO workspace_link_events (event_id,workspace_id," + "linked_workspace_id,stream_version,event_type,occurred_at_us," + "recorded_at_us,event_hash) VALUES (?,?,?,1,'workspace.linked'," + "?,?,?)", + ), + ( + "memory_capture_candidates", + "INSERT INTO memory_capture_candidates (candidate_id,workspace_id," + "idempotency_key,candidate_hash,source_kind,proposed_record_json," + "provenance_json,created_at_us) VALUES (?,?,'capture-key-0001',?," + "'native_edit','{}','{}',?)", + ), + ], +) +def test_rollback_sees_every_stream_a_client_can_write(tmp_path, table, insert): + """R-2/R-13/R-34: federation events and capture candidates count too.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + digest = "a" * 64 + linked = "ws_" + "b" * 24 + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + if table == "workspace_link_events": + connection.execute( + insert, ("evt_" + digest, workspace_id, linked, 1, 1, digest) + ) + else: + connection.execute(insert, ("cap_" + digest, workspace_id, digest, 1)) + connection.commit() + finally: + connection.close() + + with pytest.raises(MigrationV7Error) as refusal: + service.rollback(root) + assert refusal.value.code == "ROLLBACK_WOULD_HIDE_WRITES" + + async def test_an_un_migrated_v6_workspace_says_migration_is_required(tmp_path): """F-066/F-069/F-043: a non-retryable error naming the migrate command.""" @@ -526,11 +939,50 @@ async def test_an_un_migrated_v6_workspace_says_migration_is_required(tmp_path): workspace_id = _workspace_id(root) async with Client(_server(root)) as client: - response = await _invoke(client, workspace_id, "session_brief") - assert response["ok"] is False - assert response["error"]["code"] == "MIGRATION_REQUIRED" - assert response["error"]["retryable"] is False - assert "migrate-v7 --apply" in response["error"]["message"] + for tool in ("session_brief", "memory_recall", "memory_store"): + arguments: dict[str, Any] = {} + if tool == "memory_recall": + arguments = {"query": "anything"} + elif tool == "memory_store": + arguments = { + "record_type": "decision", + "content": "anything", + "idempotency_key": "v6-store-0001", + } + response = await _invoke(client, workspace_id, tool, **arguments) + assert response["ok"] is False, tool + # Store and recall are gated behind a briefing that cannot + # succeed on a v6 store, so they answer COMMUNION_REQUIRED first. + assert response["error"]["code"] in { + "MIGRATION_REQUIRED", + "COMMUNION_REQUIRED", + "SCHEMA_REJECTED", + }, (tool, response["error"]) + brief = await _invoke(client, workspace_id, "session_brief") + assert brief["error"]["code"] == "MIGRATION_REQUIRED" + assert brief["error"]["retryable"] is False + assert "migrate-v7 --apply" in brief["error"]["message"] + # The resource surface must name the same reason, not a path escape. + with pytest.raises(Exception) as resource_error: + await client.read_resource(f"memory://workspaces/{workspace_id}/rules") + assert "outside its workspace" not in str(resource_error.value) + + # The transfer boundary raises it too. It sits behind the covenant, so + # reach it the way an operator would: brief a migrated store, roll it back + # to v6 under the live session, then export. + migrated = tmp_path / "migrated" + (migrated / "src").mkdir(parents=True) + migrated_storage = _v6_database(migrated) + service = MigrationV7Service(_registry(migrated)) + service.apply(migrated) + migrated_id = _workspace_id(migrated) + async with Client(_server(migrated)) as client: + await _succeed(client, migrated_id, "session_brief") + service.rollback(migrated, discard_v7_writes=True) + assert resolve_active_database(migrated_storage).format_version == 6 + exported = await _invoke(client, migrated_id, "workspace_export") + assert exported["ok"] is False + assert exported["error"]["code"] == "MIGRATION_REQUIRED", exported["error"] def test_a_v6_rule_edited_after_an_in_place_upgrade_reaches_v7(tmp_path): @@ -588,7 +1040,7 @@ def test_a_foreign_project_trigger_is_skipped_and_counted(tmp_path): "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," "recall_topic,recall_categories,is_active,priority,created_at," "trigger_count,last_triggered) VALUES " - "(2,?,'tag_match','auth.*','authentication','[\"warning\"]',1,1,?,0,NULL)", + "(3,?,'tag_match','auth.*','authentication','[\"warning\"]',1,1,?,0,NULL)", (str(tmp_path / "another-project"), TIME), ) connection.commit() @@ -596,7 +1048,7 @@ def test_a_foreign_project_trigger_is_skipped_and_counted(tmp_path): connection.close() result = MigrationV7Service(_registry(root)).apply(root) - assert result.validation["trigger_count"] == 1 + assert result.validation["trigger_count"] == 2 assert result.validation["foreign_trigger_count"] == 1 assert any("another project path" in warning for warning in result.warnings) @@ -693,5 +1145,9 @@ def test_cli_status_leaves_a_v6_database_at_its_last_v6_version(tmp_path): connection.close() assert version == _LAST_V6_SCHEMA_VERSION assert resolve_active_database(storage).format_version == 6 - # The source is still migratable afterwards. + # The source is still migratable afterwards, and the store the ORM's own + # DDL helped build still satisfies the v7 physical-schema contract. + from daem0nmcp.verification_v7 import verify_v7 + assert MigrationV7Service(_registry(root)).apply(root).status == "activated" + assert verify_v7(storage, _workspace_id(root))["status"] == "verified" From 3c4ee7ce31618619e772aec909ec7929e65f812b Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 06:06:23 -0400 Subject: [PATCH 60/72] docs: say which tools answer MIGRATION_REQUIRED, and what an export carries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R-5: "every tool answers MIGRATION_REQUIRED" was an overclaim — store and recall are blocked behind a briefing that cannot succeed on a v6 store, so they answer COMMUNION_REQUIRED first. R-19: state positively that exports carry no host path and that the lossless v6 row lives in the migrated database rather than in the bundle. Co-Authored-By: Claude Opus 5 --- docs/v7-operations.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/v7-operations.md b/docs/v7-operations.md index 57ee845..ce11b89 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -207,8 +207,20 @@ retention, verification, atomic activation, and rollback are required. Stop the server before applying or rolling back: both commands take the storage lock, and a running server on the old pointer sees the generation -change. Until a v6 workspace is migrated, every tool answers -`MIGRATION_REQUIRED` (non-retryable) with the command to run. +change. Until a v6 workspace is migrated, `session_brief` answers +`MIGRATION_REQUIRED` (non-retryable) with the command to run, and every other +tool is blocked behind it with `COMMUNION_REQUIRED`; the resource surface and +the export/import path answer `MIGRATION_REQUIRED` too, the latter without the +command because the generic router returns the code alone. + +The migration keeps the v6 row exactly as it was inside the migrated +database — in the event log's `legacy` payload and in the retained v6 tables — +but that copy is not part of `workspace_export`, which carries the v7 +projection. Host paths are never exported: the columns that hold one are +recorded as a digest, and the usable form is derived into +`relative_file_path`. A workspace migrated before this release keeps the raw +value inside its own hash-chained events, so its exports need a re-migration +to be path-free; reads, recall, outcomes and resources do not. `migrate-v7 --apply` reports what it could not copy faithfully in `warnings` and counts it in `validation`: memory file links that pointed outside the From 0bc432bded05d39a522cee73a71908e3d3951463 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 06:07:02 -0400 Subject: [PATCH 61/72] fix: keep the preflight warning sequence typed after the PR 9 merge PR 9's `_path_safe_success` accepted `list | None`; PR 5 deleted that wrapper, so the call reaches `ResponseContext.success`, whose `warnings` is a Sequence. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/pinned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 23a8cd8..43a01e4 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -583,7 +583,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=None, ), - warnings=[draft_warning] if draft_warning is not None else None, + warnings=[draft_warning] if draft_warning is not None else (), ) token = self._dependencies.covenant_gate.issue_preflight( scope, From e227d8078f137d54091f2093ae3e9f8d2a731a36 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 10:09:33 -0400 Subject: [PATCH 62/72] fix: do not test an unhashable pair member for membership MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pair-shape check R-32 added asked `value[0] in ` before knowing `value[0]` was a string. Any ordinary two-element list whose first member is a list or dict — which the legacy-identity import path carries — raised `TypeError: unhashable type: 'list'`, and the catch-all turned it into IMPORT_INVALID. It now requires a string first, which is what a column name is, and the union is hoisted out of the recursion. The guard still rejects a `["file_path", …]` or escaping `["file_path_relative", …]` pair; `test_export_structured_fields_stay_guarded` now pins both halves, so neither the rejection nor the unhashable shapes can regress. Co-Authored-By: Claude Opus 5 --- daem0nmcp/api/v7/operations.py | 6 +++++- tests/api_v7/test_path_policy.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index 8771574..38964d2 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -102,6 +102,7 @@ _RAW_PATH_KEYS = frozenset({"file_path", "project_path", "database_path"}) _RELATIVE_PATH_KEYS = frozenset({"relative_file_path", "file_path_relative"}) _RELATIVE_PATH_ADAPTER: TypeAdapter[str] = TypeAdapter(RelativePath) +_PATH_PAIR_NAMES = _RAW_PATH_KEYS | _RELATIVE_PATH_KEYS _EVENT_COLUMNS = ( "event_id,workspace_id,stream_id,stream_kind,stream_version,event_type," "event_schema_version,occurred_at_us,recorded_at_us,actor_type,actor_id," @@ -511,7 +512,10 @@ def _reject_raw_paths(value: object) -> None: # regression must fail here rather than reach the wire. if ( len(value) == 2 - and value[0] in _RAW_PATH_KEYS | _RELATIVE_PATH_KEYS + # A column name is a string; anything else here is an ordinary + # two-element list, whose members may be unhashable. + and isinstance(value[0], str) + and value[0] in _PATH_PAIR_NAMES and isinstance(value[1], str) and value[1] ): diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py index b5b9f61..1e6ac88 100644 --- a/tests/api_v7/test_path_policy.py +++ b/tests/api_v7/test_path_policy.py @@ -240,6 +240,18 @@ def test_export_structured_fields_stay_guarded(): _reject_raw_paths([{"record": {"file_path_relative": relative}}]) with pytest.raises(CoreOperationError): _reject_raw_paths([{"record": {"file_path": "C:\\proj\\x.py"}}]) + # A migrated event carries its v6 row as ``[name, value]`` pairs, so the + # guard checks that shape too -- without mistaking an ordinary two-element + # list for one, whose members may be unhashable. + _reject_raw_paths({"legacy": {"columns": [["keywords", "x"], ["id", 1]]}}) + _reject_raw_paths({"legacy": {"columns": [[1, 2], [3, 4]]}}) + _reject_raw_paths([[{"a": 1}, {"b": 2}]]) + for pair in ( + ["file_path", "C:\\proj\\x.py"], + ["file_path_relative", "../outside/x.py"], + ): + with pytest.raises(CoreOperationError): + _reject_raw_paths({"legacy": {"columns": [pair]}}) event = { "event_id": "evt_" + "2" * 64, "event_type": "memory.created", From 5891e01fccb3362e9edc1abfce07d08c9c2c433b Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 11:09:15 -0400 Subject: [PATCH 63/72] =?UTF-8?q?fix:=20round=202=20=E2=80=94=20drop=20the?= =?UTF-8?q?=20unreachable=20resource=20branch,=20and=20stop=20racing=20the?= =?UTF-8?q?=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **CI (ubuntu 3.11).** `test_a_database_migrated_by_the_pre_fix_code_still_reads` rewrote `memory_records.content` with direct SQL and then searched for the new marker, but `memory_search_text` is served by an FTS index built at migration time from the original content, and nothing forced or awaited a rebuild. It no longer depends on re-indexing at all: one warning row now carries every shape at once — the host-absolute `file_path`, the escaping relative form, v6's unbounded tags and empty content — and is read back through `session_brief` and the warnings resource, which select by record type. That is also the reader whose failure PR 5 escalated, so the test now exercises the path the requirement came from. **MIGRATION_REQUIRED on the resource surface — wontfix in code.** `production._active_database` raises for format 6 first and that is a `RuntimeError`, so the branch I added could never match, `format_version == 6` was unreachable with that resolver, and every reader funnels exceptions through `_capture` into the deliberate `RESOURCE_UNAVAILABLE` invariant that stops a caller enumerating workspaces by reading errors. Surfacing a distinct code there would punch a hole in that invariant for no gain, so the class, its branch, the re-raise and the assertion that could not fail are deleted, and `docs/v7-operations.md` now says the resource surface answers the invariant by design and that `session_brief` is where the reason is named. **Also:** the `RecordSummary` docstring no longer claims clipping is a no-op for a v7-written record — it names the three v6 bounds a v7 write cannot violate and separates the timeline clamp, which is not a v6-only repair. The export assertion now runs over the export pages, including an events page, rather than only the recall response. The `` substitution lives in the one place that needs it (the model, plus the bounded excerpt the model does not reach); the three copies in the readers were dead, and what those readers actually needed was to stop rejecting empty content. The reactivate test says why it deletes the retained rows. **CHANGELOG:** `_lossless_row`'s host-path digest is a payload-format change, so a store migrated by an earlier development build fails `verify-v7` with "retained legacy source provenance differs" and must be migrated again from its intact v6 source. v7 is unreleased, so no shipped store is affected. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 21 +++++++ daem0nmcp/api/v7/models.py | 15 +++-- daem0nmcp/api/v7/record_operations.py | 3 +- daem0nmcp/api/v7/resource_repository.py | 27 +-------- daem0nmcp/api/v7/runtime_services.py | 4 +- docs/v7-operations.md | 9 ++- tests/test_migrate_v7_end_to_end.py | 74 +++++++++++++------------ 7 files changed, 84 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1f23f..c3b1156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,27 @@ - `start_daem0nmcp_server.bat` serves the directory you run it from (or its first argument, or an exported `DAEM0NMCP_PROJECT_ROOT`) instead of the Daem0n-MCP checkout, and `install-opencode` installs the four `.opencode/commands/` files. +- **Migration payload change — re-migrate a store migrated by an earlier + v7 development build.** `migrate-v7` no longer copies a v6 host path into the + event it writes: the columns that hold one (`file_path`, `project_path`, + `database_path`, and a `file_path_relative` that is not workspace-relative) + are recorded as a `{"$host_path": {...}}` digest, so an export bundle from a + migrated workspace no longer discloses the origin host's directory layout. + The raw value stays in the retained v6 tables and in + `.daem0nmcp/storage/migrations/v7//source.snapshot.db`. Because the + digest changes the lossless payload, a store migrated before this build fails + `verify-v7` with "retained legacy source provenance differs" and must be + migrated again from its intact v6 source (`migrate-v7 --rollback`, then + `migrate-v7 --apply`). v7 is unreleased, so this affects development stores + only. +- **Migrated v6 workspaces are usable end to end.** `session_brief`, recall, + outcomes, the four resources and `workspace_export` all work on a migrated + store; `migrate-v7` reports in `warnings` whatever it could not copy + faithfully; an un-migrated v6 workspace answers the new non-retryable + `MIGRATION_REQUIRED` naming the command instead of a retryable error with no + remedy; and `migrate-v7 --rollback` refuses with `ROLLBACK_WOULD_HIDE_WRITES` + rather than silently stranding writes made since activation (pass + `--discard-v7-writes` to accept that). ## [6.0.0] - 2026-01-29 diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index b12a635..e1a44aa 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -670,11 +670,16 @@ class RecordSummary(WireModel): """The bounded public view of a stored record. It is only ever built from storage, never parsed from a caller, so it - clips rows that predate v7's bounds -- v6 tags had no length, - uniqueness or count limit, v6 content could be empty, and v6 kept - valid time and transaction time independently -- rather than denying - every read of a migrated workspace. A v7-written record already - satisfies the bounds, so clipping is a no-op for it. + clips rows that predate v7's bounds -- v6 tags had no length, uniqueness + or count limit, and v6 content could be empty -- rather than denying every + read of a migrated workspace. A v7-written record satisfies those three, + because every stored tag input is ``UniqueTags`` and content has its own + minimum. + + The fourth clip is not a v6-only repair: a record's creation carries its + valid time, which a backdated or future ``happened_at`` legitimately puts + on either side of the transaction time it was written at, so the summary + shows the earlier of the two rather than refusing the record. ``relative_file_path`` is a containment rule rather than a bound, so it stays refused here; a stored value is normalized by diff --git a/daem0nmcp/api/v7/record_operations.py b/daem0nmcp/api/v7/record_operations.py index 2dccc82..49939f7 100644 --- a/daem0nmcp/api/v7/record_operations.py +++ b/daem0nmcp/api/v7/record_operations.py @@ -329,10 +329,11 @@ def _summary( *, include_metadata: bool = True, ) -> RecordSummary: + # v6 accepted empty content; the bounded summary renders it as the + # migration's marker rather than denying the whole read. content = row["content"] if not isinstance(content, str): raise RecordOperationError("CAPABILITY_DEGRADED") - content = content or MIGRATED_EMPTY_CONTENT tags = _parse_json(row["tags_json"], list) # A future valid-time may be recorded now; ``RecordSummary`` exposes the # transaction-time creation while the event retains the exact valid time. diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index 7fb7cf3..4626698 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -32,11 +32,7 @@ from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path from .errors import is_database_busy -from .models import ( - MIGRATED_EMPTY_CONTENT, - RecordSummary, - stored_relative_path, -) +from .models import RecordSummary, stored_relative_path from .public_ids import PublicObjectIdRepository from .resources import ( RESOURCE_FETCH_LIMIT, @@ -220,15 +216,6 @@ def __init__(self, code: str = "RESOURCE_REPOSITORY_UNAVAILABLE") -> None: super().__init__(code) -class ResourceMigrationRequiredError(RuntimeError): - """The workspace still uses the v6 storage format.""" - - code = "MIGRATION_REQUIRED" - - def __init__(self) -> None: - super().__init__(self.code) - - ActiveDatabaseResolver = Callable[[Workspace], ResolvedActiveDatabase] @@ -1013,10 +1000,6 @@ def _open_connection( raise TypeError( "active database resolver returned an invalid selection" ) - if selected.format_version == 6: - # Retrying never helps, and "outside its workspace" below would - # be a more misleading answer than naming the real reason. - raise ResourceMigrationRequiredError() if selected.format_version != 7: raise ValueError("active database is not architecture format 7") storage = normalize_resolved_path( @@ -1024,9 +1007,6 @@ def _open_connection( ) database_path = normalize_resolved_path(selected.path.resolve(strict=True)) database_path.relative_to(storage) - except ResourceMigrationRequiredError: - storage_lock.release() - raise except (OSError, RuntimeError, ValueError) as exc: storage_lock.release() raise ValueError( @@ -1137,11 +1117,10 @@ def _record_summary( if not isinstance(record_type, str) or record_type not in _PUBLIC_RECORD_TYPES: raise ValueError("record type is not public") content = row["content"] + # v6 accepted empty content; ``RecordSummary`` renders it as the + # migration's marker rather than denying the whole read. if not isinstance(content, str): raise ValueError("record content is invalid") - # v6 accepted empty content; ``RecordSummary`` renders it as the - # migration's own marker rather than denying the whole read. - content = content or MIGRATED_EMPTY_CONTENT tags = _string_list(row["tags_json"]) archived = _flag(row["archived"]) deleted = row["deleted_at_us"] is not None diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index c2b6e7c..68cb407 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -59,7 +59,6 @@ validate_directional_links, ) from .models import ( - MIGRATED_EMPTY_CONTENT, CapabilityState, CitationManifestEntry, RecordSummary, @@ -465,10 +464,11 @@ def _record_summary( evidence_status: str = "current", ) -> RecordSummary: tags = _parse_json(row["tags_json"], list, "MEMORY_RECORD_INTEGRITY_FAILED") + # v6 accepted empty content; the bounded summary renders it as the + # migration's marker rather than denying the whole read. content = row["content"] if not isinstance(content, str): raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") - content = content or MIGRATED_EMPTY_CONTENT try: return RecordSummary.model_validate( { diff --git a/docs/v7-operations.md b/docs/v7-operations.md index ce11b89..48507ff 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -209,9 +209,12 @@ Stop the server before applying or rolling back: both commands take the storage lock, and a running server on the old pointer sees the generation change. Until a v6 workspace is migrated, `session_brief` answers `MIGRATION_REQUIRED` (non-retryable) with the command to run, and every other -tool is blocked behind it with `COMMUNION_REQUIRED`; the resource surface and -the export/import path answer `MIGRATION_REQUIRED` too, the latter without the -command because the generic router returns the code alone. +tool is blocked behind it with `COMMUNION_REQUIRED`. The export/import path +answers `MIGRATION_REQUIRED` too, without the command, because the generic +router returns the code alone. The four data resources answer the single +invariant `RESOURCE_UNAVAILABLE` for every failure by design, so that a caller +cannot enumerate workspaces by reading their errors; `session_brief` is where +the reason is named. The migration keeps the v6 row exactly as it was inside the migrated database — in the event log's `legacy` payload and in the retained v6 tables — diff --git a/tests/test_migrate_v7_end_to_end.py b/tests/test_migrate_v7_end_to_end.py index 0c56897..7fc2aed 100644 --- a/tests/test_migrate_v7_end_to_end.py +++ b/tests/test_migrate_v7_end_to_end.py @@ -446,6 +446,11 @@ async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path): relative = row["record"]["file_path_relative"] assert relative is None or is_workspace_relative_path(relative), relative assert "file_path" not in row["record"] + # And the event pages, where the migrated v6 row rides as `[name, + # value]` pairs -- the place the host path used to reach the wire. + assert any(page["page_kind"] == "events" for page in pages) + for page in pages: + assert str(root) not in json.dumps(page) async def test_every_migrated_row_reads_back_through_the_server(tmp_path): @@ -497,59 +502,52 @@ async def test_a_database_migrated_by_the_pre_fix_code_still_reads( workspace_id = _workspace_id(root) absolute, _relative = _v6_file_paths(tmp_path / "other" / "helper.py", root) - marker = "spandrel" overlong = "y" * 120 tags = json.dumps([overlong, "same", "same", *[f"u{n}" for n in range(40)]]) connection = sqlite3.connect(resolve_active_database(storage).path) try: + # One warning row carrying every shape at once, so it is read back + # through `session_brief` and the warnings resource -- the reader + # whose failure PR 5 escalated, and one that selects by record type + # rather than through the lexical index, so no re-indexing is + # involved and the assertion cannot race a background rebuild. connection.execute( "UPDATE memory_records SET file_path=?,file_path_relative=?,tags_json=?," - "content=? WHERE record_type='pattern'", - (absolute, stored_relative, tags, f"The {marker} helper lives outside"), - ) - # An empty v6 content reaches the reader the same way. - connection.execute( - "UPDATE memory_records SET content='' WHERE record_type='decision'" + "content='' WHERE record_type='warning'", + (absolute, stored_relative, tags), ) connection.commit() + record_id = str( + connection.execute( + "SELECT record_id FROM memory_records WHERE record_type='warning'" + ).fetchone()[0] + ) finally: connection.close() async with Client(_server(root)) as client: brief = await _succeed(client, workspace_id, "session_brief") - assert isinstance(brief["warnings"], list) - for kind in ("warnings", "failures", "rules", "active-context"): - await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") - - hits = await _succeed( - client, - workspace_id, - "memory_search_text", - query=marker, - include_metadata=True, - ) - assert hits["items"], hits - record = hits["items"][0]["record"] + summaries = {item["record_id"]: item for item in brief["warnings"]} + assert record_id in summaries, brief["warnings"] + record = summaries[record_id] # The stored value is not workspace-relative, so no link is emitted -- # and above all the host path never reaches the wire. assert record["relative_file_path"] is None - assert absolute not in json.dumps(hits) - assert stored_relative not in json.dumps(hits) + assert absolute not in json.dumps(brief) + assert stored_relative not in json.dumps(brief) + # v6 accepted empty content; the wire needs at least one character. + assert record["excerpt"] == "" # v6 had no tag length, uniqueness or count limit; the wire has all three. assert len(record["tags"]) == 32 assert len(set(record["tags"])) == 32 assert all(len(tag) <= 80 for tag in record["tags"]) assert overlong[:80] in record["tags"] - empty = await _succeed( - client, - workspace_id, - "memory_search_text", - query="postgres", - include_metadata=True, - ) - assert empty["items"], empty - assert empty["items"][0]["record"]["excerpt"] == "" + for kind in ("warnings", "failures", "rules", "active-context"): + document = await client.read_resource( + f"memory://workspaces/{workspace_id}/{kind}" + ) + assert absolute not in document[0].text async def test_a_pre_fix_migrated_record_still_accepts_an_outcome(tmp_path): @@ -717,6 +715,10 @@ async def test_reactivating_keeps_governance_authored_through_v7(tmp_path): # Drop the retained v6 rows the compatibility write made, leaving two # streams that exist only in v7 -- the shape reconcile must not touch. + # That shape is not reachable through the tools today, because + # `rule_create`/`context_trigger_create` write the retained row and + # `rule_update` writes it back; it becomes reachable once PR 10 removes + # that compatibility write, which is what this guards. connection = sqlite3.connect(resolve_active_database(storage).path) try: connection.execute("DELETE FROM rules WHERE id>1") @@ -962,10 +964,14 @@ async def test_an_un_migrated_v6_workspace_says_migration_is_required(tmp_path): assert brief["error"]["code"] == "MIGRATION_REQUIRED" assert brief["error"]["retryable"] is False assert "migrate-v7 --apply" in brief["error"]["message"] - # The resource surface must name the same reason, not a path escape. - with pytest.raises(Exception) as resource_error: + # The resource surface deliberately answers one invariant + # (`RESOURCE_UNAVAILABLE`) for every failure, so a caller cannot + # enumerate workspaces by reading its errors; `session_brief` is + # where the reason is named. + from mcp.shared.exceptions import McpError + + with pytest.raises(McpError): await client.read_resource(f"memory://workspaces/{workspace_id}/rules") - assert "outside its workspace" not in str(resource_error.value) # The transfer boundary raises it too. It sits behind the covenant, so # reach it the way an operator would: brief a migrated store, roll it back From d8842de5b042adca51689909fd2e0edc020274de Mon Sep 17 00:00:00 2001 From: Devil <69057727+9thLevelSoftware@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:55:00 -0400 Subject: [PATCH 64/72] PR 2/12: Make the Claude Code hooks remind instead of block (#64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Brownfield run 3403b448, PR 2 of 12** — audit-and-fix stack for the v7 core (#61/#62). Base: `v7/bf-01-make-ci-green-and-pin-workspace`. ## Plan (contract) ### PR 2: Make the Claude Code hooks remind instead of block - Depends on: PR 1 - Files: daem0nmcp/claude_hooks/pre_edit.py, daem0nmcp/claude_hooks/post_edit.py, daem0nmcp/claude_hooks/stop.py, daem0nmcp/claude_hooks/_client.py, daem0nmcp/claude_hooks/install.py, hooks/daem0n_pre_edit_hook.py, hooks/daem0n_post_edit_hook.py, hooks/daem0n_prompt_hook.py, hooks/daem0n_stop_hook.py, tests/test_claude_hooks_pre_edit.py, tests/test_claude_hooks_stop.py, tests/test_claude_hooks_post_edit.py, tests/test_claude_hooks_install.py, tests/api_v7/test_docs_hooks.py, docs/release/v7/requirements.json, docs/release/v7/inventory.md, README.md, Summon_Daem0n.md, docs/v7-operations.md - Traces to: FP-4, FP-7, F-009, F-019, F-045, F-050, F-042, F-044, F-084, F-052, F-037, F-038, A-025, A-031, A-036, UD-1 - Risk: low - Description (KD-1): 1. **`pre_edit.py` rewrite.** Stdlib + `_client.read_hook_event` only. - Take the project from `event["cwd"]`, falling back to `CLAUDE_PROJECT_DIR`. - With no `.daem0nmcp/` there, exit 0 silently. - Otherwise print `{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":""}}` and exit 0. - Never set `permissionDecision`. - Delete `async_main`, `PreEditResult` and `handle_pre_edit`. 2. **`post_edit.py`.** Import `edit_host` lazily, only when the bridge credential env is configured. 3. **`stop.py`** (R-7). - Read `transcript_path` and `session_id` from the stdin event, and parse transcript records nested under `message`. - Emit the reminder only as a user-visible `systemMessage` JSON field or as plain stdout. Record in the PR which one Claude Code displays. - Never emit `"decision"` and never exit 2. - It still never writes memory. 4. **`_client.py`.** Delete `get_tool_input` and `get_file_path_from_input`. 5. **Uninstall cleanup** (F-038). - `uninstall-claude-hooks --project-path` strips the Daem0n env keys it added to `/.claude/settings.local.json`. - With `--remove-credentials`, it also deletes `~/.daem0nmcp/edit-bridges//`. - Both `install-claude-hooks` and `uninstall-claude-hooks` remove any `hooks/daem0n_*` command entries from `~/.claude/settings.json` (R-20; `install.py:45-48` already detects them). 6. **Legacy scripts** (R-13/R-20). Replace each `hooks/daem0n_*.py` with a stdlib-only stub. The stub prints `Daem0n: this manual hook is deprecated; run python -m daem0nmcp.cli install-claude-hooks` to stderr and exits 0. Delete the README "Manual Hooks (Legacy)" block. The PR description and CHANGELOG note tell manual-hook users to re-run `install-claude-hooks`. 7. **Doc sections.** Only the hook sections of README §Claude Code Integration, Summon §9 and v7-operations §Claude Code: one table of the 6 hooks (event, what it does, input channel, reminds only), the user-level install location, and what pairing writes. 8. **Release inventory** (R-2/R-22). Run `python scripts/v7_release_inventory.py --write` and commit the regenerated `requirements.json`/`inventory.md`, because hook entry points changed. 9. **Tests** (F-052). - stdin-driven `main()` subprocess tests for `pre_edit`: non-Daem0n dir → exit 0 and empty stdout; Daem0n project → exit 0 plus the reminder JSON. - An import-set test: no `daem0nmcp.api`/`daem0nmcp.retrieval` module loaded. - `stop` driven by a nested-`message` JSONL fixture, asserting its output never contains `"decision"`. - Each legacy stub fed a PreToolUse payload exits 0. - The uninstall test. - `test_docs_hooks.py` updated. - Acceptance: - A Claude-style Write event piped into `pre_edit` from a non-Daem0n temp dir exits 0 with empty stdout. From a Daem0n project it exits 0 with the reminder JSON. - `pre_edit` median runtime is < 150 ms (was 583 ms). - No Daem0n hook and no legacy stub can exit 2 (the tests above). - The release-inventory CI job is green. - The PR states whether `additionalContext` was confirmed visible to the model in a live session (A-036). #### Carry-over from PR 1 review (orchestrator, binding) - PR 1 (b881ca6) moved local edit-bridge sockets to `edit-bridges/run//b-<24hex>.sock`. `uninstall-claude-hooks --remove-credentials` (step 5) must also remove that socket directory. --- ## What landed # PR 2 summary: Make the Claude Code hooks remind instead of block (Saved by the orchestrator from the implementer's reply.) Branch `v7/bf-02-make-the-claude-code-hooks-remin`, base PR 1 tip f7f024a. Worktree `D:\Daem0n-MCP\.claude\worktrees\agent-ab437c58b407838c4`. - 220f98a `fix: make the Claude Code hooks remind instead of block` - a9e7d7a `docs: describe the six remind-only Claude Code hooks and pairing` (final) ## Files changed - `daem0nmcp/claude_hooks/pre_edit.py` (rewritten); `stop.py`, `post_edit.py`, `_client.py`, `install.py`. - Outside plan file list: `daem0nmcp/cli.py` (`--remove-credentials`, `--project-path` to uninstall), `daem0nmcp/edit_host.py` (extracted `local_bridge_layout()`). - `hooks/daem0n_{pre_edit,post_edit,prompt,stop}_hook.py` → exit-0 deprecation stubs; `hooks/settings.json.example` deleted. - Tests: `test_claude_hooks_{pre_edit,stop,post_edit,install,client}.py`, `api_v7/test_docs_hooks.py`; bridge tests `test_edit_host.py`, `test_edit_bridge_transport.py`, `api_v7/test_process_edit_capture.py`; new helper `tests/native_edit_host.py`. - Docs: `README.md`, `Summon_Daem0n.md`, `docs/v7-operations.md`, `CHANGELOG.md`. ## What and why 1. **pre_edit**: stdlib + `read_hook_event` only; project from `event["cwd"]` then `CLAUDE_PROJECT_DIR`; no `.daem0nmcp/` → exit 0 silent; else prints PreToolUse `additionalContext` reminder (memory_recall_file + memory_preflight) and exits 0; path omitted if outside project; never sets `permissionDecision`; exceptions swallowed → exit 0. Deleted `async_main`, `PreEditResult`, `handle_pre_edit`. Lazy `asyncio` import in `_client.run_async`. Median runtime 87 ms (was 503 ms), 21 subprocess runs on Windows. 2. **post_edit**: `edit_host`/`native_edit` imported lazily; only runs when `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE` is set. 3. **stop**: reads `cwd`, `transcript_path`, `session_id` from stdin; state keyed by session_id; unwraps nested `message` records; emits `{"systemMessage": ...}` exit 0; never `decision`, never writes memory. Live `claude -p --output-format stream-json` run showed the systemMessage as an informational system event. 4. **_client**: deleted `get_tool_input`, `get_file_path_from_input`; env parsing folded into `get_command_from_input` (pre_bash needs it until PR 10). 5. **Uninstall**: `cli --project-path P uninstall-claude-hooks [--remove-credentials] [--dry-run]` strips pairing env keys from `/.claude/settings.local.json` (credential file, runtime dir, mode, workspace-binding file, `DAEM0NMCP_PROJECT_ROOT`, `DAEM0NMCP_STORAGE_PATH`), keeping others; `--remove-credentials` deletes `edit-bridges//` and `edit-bridges/run//` (PR 1 carry-over). Legacy `hooks/daem0n_*` entries removed via `_is_daem0n_entry` (test added). 6. **Legacy scripts**: import only `sys`, print deprecation to stderr, exit 0. README "Manual Hooks (Legacy)" block deleted; CHANGELOG tells manual-hook users to re-run `install-claude-hooks`. 7. **Docs**: README, Summon §9, v7-operations share one six-hook table (event, behaviour, input channel, reminds-only, install location, what pairing writes); states pre_bash is inert (reads unset `TOOL_INPUT`) and post_edit hooks are no-ops. 8. **Release inventory**: `--write` produced no diff; `--check` valid (75 tools, 10 resources, 571 requirements). ## Tests and results - New pre_edit subprocess tests: non-Daem0n dir silent exit 0; Daem0n project exit 0 + reminder JSON with no permissionDecision; CLAUDE_PROJECT_DIR fallback; outside path and garbage stdin exit 0; import set excludes `daem0nmcp.api`, `retrieval`, `edit_host`, `database`, `sqlalchemy`, `pydantic`. - Stop: `main()` over stdin with nested-message JSONL; asserts no `"decision"`, systemMessage mentions memory_store/memory_record_outcome, no storage written, state file created; silent outside Daem0n project. - post_edit: unpaired project exits 0 without importing edit_host; paired project with no consumed edit exits 0. - Install/uninstall: legacy entries removed; project cleanup dry-run, env keys, both credential dirs; credentials kept without the flag. - `test_docs_hooks.py`: stubs sys-only, never exit(2); each stub fed a PreToolUse payload exits 0 with deprecation on stderr and no workspace mutation. - Hook + bridge tests: 105 passed (Windows); 86 passed (Docker python:3.12 Linux). - Full suite (Windows): 2799 passed, 121 skipped, 13 min 08 s. - ruff check/format clean; mypy api/v7 clean (also clean on claude_hooks and edit_host.py). - A-036 live check: `claude -p` (Claude Code 2.1.277) with only the new pre_edit hook in a temp dir with `.daem0nmcp/`: the Write went through and the model quoted the additionalContext. Confirmed. ## Deviations 1. Deleting `handle_pre_edit` broke bridge integration tests that used it as the native host → test-only driver `tests/native_edit_host.py::drive_native_edit` (raises instead of swallowing; two test_edit_host "fails before network" tests now assert `pytest.raises(ValueError)` and `calls == 0`). 2. `edit_host.py`: extracted `local_bridge_layout()` so uninstall computes the same dirs as provisioning; provisioning unchanged. 3. `cli.py`: `--remove-credentials`, `--project-path` passed through. 4. Deleted `hooks/settings.json.example`. 5. Deleted old pre_edit tests asserting blocking/permissive mode and `test_remote_mode_without_workspace_binding_fails_recoverably` (config error still covered by `EditHostConfig.from_environment`). ## Follow-ups (not fixed) - Claude-side edit bridge now unreachable (only pre_edit created bridge requests) → post_edit_preflight/post_edit are no-ops in Claude Code (documented). Fits UD-13 follow-up; pairing on install could be dropped then. - pre_bash still reads `TOOL_INPUT`; `block()` still in `_client` (PR 10). `run_hook_safely` maps exit 2 to 0. - Uninstall doesn't remove a remote pairing's binding file. - `daem0nmcp/__init__.py` `importlib.metadata.version()` costs ~36 ms per hook start. - README historical v6 section still says pre_edit "fails closed" (PR 12 scope). - `.gitignore` lacks `.venv/`. ## Note for PR description Manual-hook users: `hooks/daem0n_*.py` now only print a deprecation notice and exit 0. Re-run `python -m daem0nmcp.cli install-claude-hooks`; it also removes the old entries. ## Round 2 134b724 (hooks hardening: find_project_root walk-up stopping at home; lexical relative_project_path rejecting UNC/NUL/outside; json.dumps path; uninstall reads-before-writes with partial-success reporting; utf-8-sig settings reads; install refuses malformed ~/.claude/settings.json; pairing-marker-gated env strip; --project-path after subcommand; CLI cwd default; reject_linked_ancestry before rmtree; stop reads transcript tail, user-facing systemMessage, prunes state >7 days) and 27d1682 (docs: OpenCode edit gating gone; project detection). 26 items: 23 addressed, 3 wontfix (R-14 pre-existing bootstrap flake → PR 8; R-24 UD-13 follow-up list; R-25 plan amendments done by orchestrator). Security reviewer re-verified R-19..R-22 resolved. CI run 35422498587 green on all 9 legs + lint/type-check/release-inventory. UD-13 follow-up items: stop pairing on install; unregister post_edit/post_edit_preflight and strip them on install; delete tests/native_edit_host.py with the bridge; surface pre_edit stdout as OpenCode reminder; drop failClosed (incl. 5 s timeout block); fix daem0n.ts header/comments (lines 6, 216). --- ## Review # PR 2 review — merged (round 2: all addressed; security re-verified R-19..R-22; CI run 35422498587 green) Reviewer slots: general, general-2, tests, security, plan-alignment (effort 5). Item statuses: 23 addressed, 3 wontfix. CI: https://github.com/9thlevelsoftware/Daem0n-MCP/actions/runs/35422498587 (green) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 7 + README.md | 81 ++++---- Summon_Daem0n.md | 30 ++- daem0nmcp/claude_hooks/_client.py | 78 +++++-- daem0nmcp/claude_hooks/install.py | 186 ++++++++++++++--- daem0nmcp/claude_hooks/post_edit.py | 18 +- daem0nmcp/claude_hooks/pre_edit.py | 193 ++++------------- daem0nmcp/claude_hooks/stop.py | 131 +++++++----- daem0nmcp/cli.py | 16 +- daem0nmcp/edit_host.py | 35 +++- docs/v7-operations.md | 29 +++ hooks/daem0n_post_edit_hook.py | 142 +------------ hooks/daem0n_pre_edit_hook.py | 102 +-------- hooks/daem0n_prompt_hook.py | 52 +---- hooks/daem0n_stop_hook.py | 150 +------------- hooks/settings.json.example | 70 ------- tests/api_v7/test_docs_hooks.py | 175 +++++----------- tests/api_v7/test_process_edit_capture.py | 10 +- tests/native_edit_host.py | 121 +++++++++++ tests/test_claude_hooks_cli.py | 79 +++++++ tests/test_claude_hooks_client.py | 22 +- tests/test_claude_hooks_install.py | 206 +++++++++++++++++++ tests/test_claude_hooks_post_edit.py | 153 +++++++------- tests/test_claude_hooks_pre_edit.py | 239 +++++++++++++--------- tests/test_claude_hooks_stop.py | 108 +++++++++- tests/test_edit_bridge_transport.py | 10 +- tests/test_edit_host.py | 56 +++-- 27 files changed, 1340 insertions(+), 1159 deletions(-) delete mode 100644 hooks/settings.json.example create mode 100644 tests/native_edit_host.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c67c0..dca386b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ ### Added - Reproducible v7 core foundation with opt-in dependency profiles and lazy capability reporting. +- `uninstall-claude-hooks` strips the pairing env keys from the project's `.claude/settings.local.json` (the project is `--project-path` or the current directory, and only a project the installer paired is touched); `--remove-credentials` also deletes its local edit-bridge credential and socket directories. + +### Changed +- Claude Code hooks remind instead of block. `pre_edit` adds a one-line `memory_recall_file`/`memory_preflight` reminder to the model's context and never denies an edit; `stop` reads its stdin event, shows suggestions as a `systemMessage` and never keeps the agent running. +- OpenCode edits are no longer gated either: the OpenCode plugin runs the same `pre_edit` hook, which now always allows the edit. The plugin does not yet show the reminder; that rework comes with the edit-bridge removal. +- `install-claude-hooks` and `uninstall-claude-hooks` refuse to touch a `~/.claude/settings.json` that is not valid JSON, instead of replacing it. +- The manual `hooks/daem0n_*.py` scripts are deprecated stubs that print a notice and exit 0, and `hooks/settings.json.example` is removed. If you installed them by hand, re-run `python -m daem0nmcp.cli install-claude-hooks`; it also removes the old entries. ## [6.0.0] - 2026-01-29 diff --git a/README.md b/README.md index daad047..c2ca8d7 100644 --- a/README.md +++ b/README.md @@ -608,55 +608,54 @@ Use `system_health` for bounded diagnostics. The complete protocol lives in ## Claude Code Integration -### Automated Hook Installation (Recommended) +### Hook Installation ```bash python -m daem0nmcp.cli install-claude-hooks ``` -This registers 5 hook modules in `~/.claude/settings.json`: +This writes six hook entries to the user-level `~/.claude/settings.json`, so +they run in every project; each runs `"" -m daem0nmcp.claude_hooks.` +with the interpreter you installed from. A hook only acts inside a Daem0n +project: the session directory, `CLAUDE_PROJECT_DIR`, or one of their parent +directories contains `.daem0nmcp/` (your home directory does not count, since +Daem0n keeps its own state in `~/.daem0nmcp/`). Other hooks in that file are kept, and any old +`hooks/daem0n_*.py` entries are removed. + +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | + +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. + +**What pairing writes.** The installer also pairs the current project (or +`--project-path`) with the local edit bridge: + +- an owner-only credential at `~/.daem0nmcp/edit-bridges//credential.json` + (the secret is never written into the project), with the bridge socket under + `~/.daem0nmcp/edit-bridges/run//`; +- the env keys `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, + `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, `DAEM0NMCP_PROJECT_ROOT` and + `DAEM0NMCP_STORAGE_PATH` in `/.claude/settings.local.json`. + +To remove the hooks, the project's pairing env keys (only if the installer +paired it) and, optionally, its credential (the project defaults to the current +directory): -| Event | Hook | Purpose | -|-------|------|---------| -| `SessionStart` | `session_start` | Scoped `session_brief` instruction | -| `PreToolUse` (Edit/Write/NotebookEdit) | `pre_edit` | Fail-closed v7 preflight instruction | -| `PreToolUse` (Bash) | `pre_bash` | Rule enforcement on commands | -| `PostToolUse` (Edit/Write) | `post_edit` | Replay-safe memory suggestion | -| `Stop`/`SubagentStop` | `stop` | Outcome reminder without direct memory writes | - -To remove: `python -m daem0nmcp.cli uninstall-claude-hooks` - -### Manual Hooks (Legacy) - -Add to `.claude/settings.json`: -```json -{ - "hooks": { - "PreToolUse": [{ - "matcher": "Edit|Write|NotebookEdit", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_pre_edit_hook.py\"" - }] - }], - "PostToolUse": [{ - "matcher": "Edit|Write", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_post_edit_hook.py\"" - }] - }], - "Stop": [{ - "matcher": "", - "hooks": [{ - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - }] - }] - } -} +```bash +python -m daem0nmcp.cli uninstall-claude-hooks [--project-path ] [--remove-credentials] ``` +The old manual scripts in `hooks/` are deprecated stubs that only print a +notice and exit 0. If your settings still point at them, re-run +`install-claude-hooks`. + ### Protocol Skill A Claude Code skill is included at `.claude/skills/daem0nmcp-protocol/SKILL.md` that enforces the memory protocol automatically. diff --git a/Summon_Daem0n.md b/Summon_Daem0n.md index eafc434..c8a0431 100644 --- a/Summon_Daem0n.md +++ b/Summon_Daem0n.md @@ -167,18 +167,34 @@ tools, never through a direct database, script, or memory-writing CLI command. ## 9. Hook behavior -The root hook templates are advisory and read-only. The pre-edit hook fails -closed because a standalone process cannot validate a scoped capability. It -directs Claude Code to the exact `memory_recall` and `memory_preflight` calls. -Post-edit and Stop hooks print replay-safe `memory_store` and -`memory_record_outcome` suggestions; they do not mutate Daem0n storage. - -Install the packaged hook configuration with: +Install the packaged hooks with: ```bash python -m daem0nmcp.cli install-claude-hooks ``` +This writes six entries to the user-level `~/.claude/settings.json`. The hooks +remind; they never block an edit, a command, or the end of a turn, and none of +them writes Daem0n memory. + +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | + +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. + +Run from a project, the installer also pairs that project with the local edit +bridge: an owner-only credential under `~/.daem0nmcp/edit-bridges//` +and bridge path env keys in `/.claude/settings.local.json`. The old +`hooks/daem0n_*.py` scripts are deprecated stubs; re-run `install-claude-hooks` +if your settings still point at them. + ## 10. Migration reference The generated source of truth for renamed or split v6 capabilities is diff --git a/daem0nmcp/claude_hooks/_client.py b/daem0nmcp/claude_hooks/_client.py index fcd1f2d..c123e22 100644 --- a/daem0nmcp/claude_hooks/_client.py +++ b/daem0nmcp/claude_hooks/_client.py @@ -5,7 +5,6 @@ Daem0n-MCP's database, memory, and rules from hook scripts. """ -import asyncio import json import os import sys @@ -34,6 +33,59 @@ def read_hook_event() -> dict[str, Any]: return value if isinstance(value, dict) else {} +def find_project_root(event: dict[str, Any]) -> Path | None: + """Return the nearest Daem0n project containing the event cwd. + + Tries ``cwd`` (which follows ``cd`` in Bash), then ``CLAUDE_PROJECT_DIR``, + walking up to the first ancestor with ``.daem0nmcp/``. The home directory + never counts: Daem0n keeps its own global state in ``~/.daem0nmcp/``. + """ + home = Path.home() + for raw in (event.get("cwd"), os.environ.get("CLAUDE_PROJECT_DIR")): + if not isinstance(raw, str) or not raw: + continue + start = Path(os.path.abspath(raw)) + for candidate in (start, *start.parents): + if candidate == home: + break + if (candidate / ".daem0nmcp").is_dir(): + return candidate + return None + + +def _is_unc(path: str) -> bool: + return path.startswith(("\\\\", "//")) + + +def relative_project_path( + project: Path, raw: object, base: object = None +) -> str | None: + """Return *raw* (relative to *base*, default *project*) relative to *project*. + + The result is in POSIX form, or None when *raw* is not inside *project*. + + Purely lexical: the path comes from the model, and resolving it on + Windows could open a ``\\\\host\\share`` path (SMB authentication) before + the user has approved anything. + """ + if not isinstance(raw, str) or not raw or "\0" in raw or _is_unc(raw): + return None + root = os.path.normpath(str(project)) + start = base if isinstance(base, str) and base else root + full = os.path.normpath(os.path.join(root, start, raw)) + if _is_unc(full): + return None + try: + inside = os.path.commonpath( + [os.path.normcase(root), os.path.normcase(full)] + ) == os.path.normcase(root) + except ValueError: # different drives, or mixed absolute/relative + return None + if not inside or os.path.normcase(full) == os.path.normcase(root): + return None + return Path(os.path.relpath(full, root)).as_posix() + + def get_project_path() -> str | None: """ Detect the project path from environment variables. @@ -75,28 +127,18 @@ def get_managers(project_path: str): def run_async(coro) -> Any: """Run an async coroutine synchronously.""" + import asyncio # lazy: keeps the stdlib-only pre_edit hook fast + return asyncio.run(coro) -def get_tool_input() -> dict: - """Parse TOOL_INPUT env var as JSON, returning {} on failure.""" - raw = os.environ.get("TOOL_INPUT", "{}") +def get_command_from_input() -> str | None: + """Extract command from the legacy TOOL_INPUT env var (for Bash hooks).""" try: - return json.loads(raw) + data = json.loads(os.environ.get("TOOL_INPUT", "{}")) except (json.JSONDecodeError, TypeError): - return {} - - -def get_file_path_from_input() -> str | None: - """Extract file_path or notebook_path from tool input.""" - data = get_tool_input() - return data.get("file_path") or data.get("notebook_path") - - -def get_command_from_input() -> str | None: - """Extract command from tool input (for Bash hooks).""" - data = get_tool_input() - return data.get("command") + return None + return data.get("command") if isinstance(data, dict) else None def block(message: str) -> NoReturn: diff --git a/daem0nmcp/claude_hooks/install.py b/daem0nmcp/claude_hooks/install.py index cbfa4a6..37fbd65 100644 --- a/daem0nmcp/claude_hooks/install.py +++ b/daem0nmcp/claude_hooks/install.py @@ -7,25 +7,57 @@ import argparse import json +import shutil import sys from pathlib import Path from typing import Any -from ..edit_host import provision_client_bridge_installation +from ..edit_host import ( + local_bridge_layout, + provision_client_bridge_installation, +) +from ..protected_files import reject_linked_ancestry + +# Env keys only pairing writes into /.claude/settings.local.json. +# Pairing always writes the credential key, so it marks a paired project. +_PAIRING_MARKER = "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" +_BRIDGE_ENV_KEYS = ( + _PAIRING_MARKER, + "DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR", + "DAEM0NMCP_EDIT_BRIDGE_MODE", + "DAEM0NMCP_EDIT_HOST_WORKSPACE_BINDING_FILE", +) +# General Daem0n keys that pairing also sets; removed only from paired projects. +_SHARED_ENV_KEYS = ("DAEM0NMCP_PROJECT_ROOT", "DAEM0NMCP_STORAGE_PATH") def _settings_path() -> Path: return Path.home() / ".claude" / "settings.json" +def _read_json_object(path: Path) -> dict[str, Any]: + """Read a JSON object (a UTF-8 BOM is fine); raise ValueError otherwise.""" + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{path} is not readable JSON ({exc})") from exc + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + def _read_settings() -> dict[str, Any]: + """Read ``~/.claude/settings.json``; raise ValueError rather than lose it.""" path = _settings_path() - if path.exists(): - try: - return json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return {} - return {} + if not path.exists(): + return {} + settings = _read_json_object(path) + hooks = settings.get("hooks", {}) + if not isinstance(hooks, dict) or not all( + isinstance(entries, list) for entries in hooks.values() + ): + raise ValueError(f"{path}: 'hooks' must map each event to a list") + return settings def _write_settings(data: dict[str, Any]) -> None: @@ -36,8 +68,11 @@ def _write_settings(data: dict[str, Any]) -> None: def _is_daem0n_entry(entry: dict) -> bool: """Check if a hook entry belongs to Daem0n (current or legacy).""" - for hook in entry.get("hooks", []): - cmd = hook.get("command", "") + hooks = entry.get("hooks") if isinstance(entry, dict) else None + for hook in hooks if isinstance(hooks, list) else []: + cmd = hook.get("command") if isinstance(hook, dict) else None + if not isinstance(cmd, str): + continue if "daem0nmcp.claude_hooks" in cmd: return True # Legacy hook scripts @@ -141,12 +176,7 @@ def _configure_project_bridge( environment: dict[str, str], ) -> Path: path = project_root / ".claude" / "settings.local.json" - if path.exists(): - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError("project-local Claude settings must be an object") - else: - value = {} + value = _read_json_object(path) if path.exists() else {} configured = value.setdefault("env", {}) if not isinstance(configured, dict): raise ValueError("project-local Claude env settings must be an object") @@ -175,7 +205,10 @@ def install_claude_hooks( Returns (success, message). """ - settings = _read_settings() + try: + settings = _read_settings() + except ValueError as exc: + return False, f"{exc}; fix or move it and retry. Nothing was changed." new_defs = _build_hook_definitions() project_root = ( None if project_path is None else Path(project_path).resolve(strict=True) @@ -236,18 +269,75 @@ def install_claude_hooks( return True, message -def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: +def _strip_pairing_env(path: Path) -> dict[str, Any] | None: + """Return *path*'s settings without the pairing env keys, or None if unpaired.""" + if not path.is_file(): + return None + value = _read_json_object(path) + configured = value.get("env") + if not isinstance(configured, dict) or _PAIRING_MARKER not in configured: + return None + for key in _BRIDGE_ENV_KEYS + _SHARED_ENV_KEYS: + configured.pop(key, None) + if not configured: + del value["env"] + return value + + +def uninstall_claude_hooks( + dry_run: bool = False, + *, + project_path: str | Path | None = None, + remove_credentials: bool = False, + bridge_config_root: Path | None = None, +) -> tuple[bool, str]: """ Remove all Daem0n Claude Code hooks. - Preserves all other hooks. Cleans up empty event lists. + Preserves all other hooks. Cleans up empty event lists. With + *project_path*, also strips the pairing env keys from that project's + ``.claude/settings.local.json``; with *remove_credentials*, also deletes + the project's local edit-bridge credential and socket directories. - Returns (success, message). + Returns (success, message); the message lists everything that was done, + including after a partial failure. """ - settings = _read_settings() + try: + settings = _read_settings() + except ValueError as exc: + return False, f"{exc}; fix or move it and retry. Nothing was changed." + + # Read everything that can fail before writing anything. + notes: list[str] = [] + ok = True + local_path: Path | None = None + local_value: dict[str, Any] | None = None + directories: list[Path] = [] + if project_path is not None: + project_root = Path(project_path).resolve(strict=False) + local_path = project_root / ".claude" / "settings.local.json" + try: + local_value = _strip_pairing_env(local_path) + except ValueError as exc: + notes.append(f"Left {local_path} unchanged: {exc}") + ok = False + if remove_credentials: + try: + _, _, *layout = local_bridge_layout(project_root, bridge_config_root) + except (OSError, ValueError) as exc: + notes.append( + f"Could not locate edit bridge credentials for {project_root} " + f"({exc}); delete its directory under " + "~/.daem0nmcp/edit-bridges/ by hand." + ) + ok = False + else: + directories = [path for path in layout if path.is_dir()] + elif remove_credentials: + notes.append("No project given; edit bridge credentials kept.") + hooks_section = settings.get("hooks", {}) removed_events: list[str] = [] - for event in list(hooks_section.keys()): entries = hooks_section[event] filtered = [e for e in entries if not _is_daem0n_entry(e)] @@ -258,21 +348,44 @@ def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: else: del hooks_section[event] + lines: list[str] = [] if not removed_events: - return True, "No Daem0n hooks found to remove." - - settings["hooks"] = hooks_section - - if dry_run: + lines.append("No Daem0n hooks found to remove.") + elif dry_run: + settings["hooks"] = hooks_section formatted = json.dumps(settings, indent=2) - return True, f"[dry-run] Would write to {_settings_path()}:\n{formatted}" - - try: - _write_settings(settings) - except OSError as exc: - return False, f"Failed to write settings: {exc}" + lines.append(f"[dry-run] Would write to {_settings_path()}:\n{formatted}") + else: + settings["hooks"] = hooks_section + try: + _write_settings(settings) + except OSError as exc: + return False, f"Failed to write settings: {exc}" + lines.append(f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}") + lines.extend(notes) + + prefix = "[dry-run] Would remove" if dry_run else "Removed" + if local_path is not None and local_value is not None: + try: + if not dry_run: + local_path.write_text( + json.dumps(local_value, indent=2) + "\n", encoding="utf-8" + ) + lines.append(f"{prefix} edit bridge settings from {local_path}") + except OSError as exc: + lines.append(f"Could not update {local_path}: {exc}") + ok = False + for directory in directories: + try: + reject_linked_ancestry(directory) + if not dry_run: + shutil.rmtree(directory) + lines.append(f"{prefix} {directory}") + except OSError as exc: + lines.append(f"Could not remove {directory}: {exc}") + ok = False - return True, f"Removed Daem0n hooks from: {', '.join(sorted(removed_events))}" + return ok, "\n".join(lines) if __name__ == "__main__": @@ -285,10 +398,15 @@ def uninstall_claude_hooks(dry_run: bool = False) -> tuple[bool, str]: parser.add_argument("--remote-url") parser.add_argument("--remote-ca-file") parser.add_argument("--remote-origin") + parser.add_argument("--remove-credentials", action="store_true") args = parser.parse_args() if args.uninstall: - ok, msg = uninstall_claude_hooks(dry_run=args.dry_run) + ok, msg = uninstall_claude_hooks( + dry_run=args.dry_run, + project_path=args.project_path, + remove_credentials=args.remove_credentials, + ) else: ok, msg = install_claude_hooks( dry_run=args.dry_run, diff --git a/daem0nmcp/claude_hooks/post_edit.py b/daem0nmcp/claude_hooks/post_edit.py index b3b3320..3d15628 100644 --- a/daem0nmcp/claude_hooks/post_edit.py +++ b/daem0nmcp/claude_hooks/post_edit.py @@ -7,9 +7,11 @@ from collections.abc import Mapping from typing import Any -from ..edit_host import EditHostConfig, EditHostStateStore, native_edit_capture_body from ._client import read_hook_event -from .native_edit import configured_native_edit_tools, native_edit_relative_paths + +# Pairing writes this into the project's Claude settings; without it there is +# no edit bridge, so the heavy bridge modules are never imported. +_CREDENTIAL_ENV = "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" def _ok(response: object) -> bool: @@ -20,6 +22,16 @@ def _ok(response: object) -> bool: def handle_post_edit(event: Mapping[str, Any], project_path: str) -> bool: """Stage only a bounded host-generated candidate for a consumed request.""" try: + from ..edit_host import ( + EditHostConfig, + EditHostStateStore, + native_edit_capture_body, + ) + from .native_edit import ( + configured_native_edit_tools, + native_edit_relative_paths, + ) + native_session_id = event["session_id"] native_request_id = event["tool_use_id"] tool_name = event["tool_name"] @@ -69,7 +81,7 @@ def handle_post_edit(event: Mapping[str, Any], project_path: str) -> bool: def main() -> None: event = read_hook_event() path = event.get("cwd") - if isinstance(path, str) and path: + if isinstance(path, str) and path and os.environ.get(_CREDENTIAL_ENV): handle_post_edit(event, path) sys.exit(0) diff --git a/daem0nmcp/claude_hooks/pre_edit.py b/daem0nmcp/claude_hooks/pre_edit.py index a446d6c..b182b89 100644 --- a/daem0nmcp/claude_hooks/pre_edit.py +++ b/daem0nmcp/claude_hooks/pre_edit.py @@ -1,161 +1,56 @@ -"""Claude Code native-edit bridge adapter.""" +"""Claude Code PreToolUse hook for edits: remind, never block. + +Stdlib only (plus the ``_client`` helpers) so it stays fast on every edit. Outside +a Daem0n project it is silent; inside one it adds a one-line reminder to the +model's context and always exits 0. +""" from __future__ import annotations -import os +import json import sys -from collections.abc import Mapping -from datetime import datetime -from typing import Any - -from ..edit_host import ( - CreatedHostSession, - EditHostConfig, - EditHostStateStore, - PendingEditBinding, -) -from ._client import block, read_hook_event -from .native_edit import configured_native_edit_tools, native_edit_request - - -class PreEditResult: - def __init__(self, allowed: bool, message: str) -> None: - self.allowed, self.message = allowed, message - - -async def async_main(project_path: str, file_path: str) -> PreEditResult: - """Compatibility entry point; standalone calls cannot forge host sessions.""" - del project_path, file_path - return PreEditResult(False, "EDIT_BRIDGE_UNAVAILABLE") - - -def _expires(value: object) -> datetime: - if not isinstance(value, str): - raise ValueError - return datetime.fromisoformat(value.removesuffix("Z") + "+00:00") - -def _data(response: object) -> Mapping[str, Any]: - body = getattr(response, "body", None) - if ( - not isinstance(body, Mapping) - or body.get("ok") is not True - or not isinstance(body.get("data"), Mapping) - ): - raise RuntimeError - return body["data"] - - -def handle_pre_edit(event: Mapping[str, Any], project_path: str) -> PreEditResult: - try: - native_session_id, native_request_id, tool_name, tool_input = ( - event["session_id"], - event["tool_use_id"], - event["tool_name"], - event["tool_input"], - ) - if ( - not isinstance(native_session_id, str) - or not isinstance(native_request_id, str) - or not isinstance(tool_name, str) - or not isinstance(tool_input, Mapping) - ): - raise ValueError - config = EditHostConfig.from_environment(os.environ) - workspace_id = config.workspace_id(project_path) - store = EditHostStateStore(config) - client = config.build_client() - edit = native_edit_request( - project_path=project_path, - tool_name=tool_name, - tool_input=tool_input, - configured_tools=configured_native_edit_tools(os.environ), - ) - - def create() -> CreatedHostSession: - data = _data(client.call("/v1/sessions", {"workspace_id": workspace_id})) - return CreatedHostSession( - str(data["host_session_id"]), _expires(data["expires_at"]) - ) - - host = store.load_or_create_session( - workspace_id=workspace_id, - native_session_id=native_session_id, - credential_id=config.identity.credential_id, - create=create, - ) - pending = store.get_pending( - workspace_id=workspace_id, - native_session_id=native_session_id, - credential_id=config.identity.credential_id, - edit_hash=edit.edit_hash, - ) - wire_edit = { - "tool_name": edit.tool_name, - "arguments": dict(edit.arguments), - "preimages": [item.canonical() for item in edit.preimages], +from ._client import find_project_root, read_hook_event, relative_project_path + + +def reminder(event: dict) -> str | None: + """Return the reminder JSON for this event, or None to stay silent.""" + project = find_project_root(event) + if project is None: + return None + tool_input = event.get("tool_input") + raw = ( + tool_input.get("file_path") or tool_input.get("notebook_path") + if isinstance(tool_input, dict) + else None + ) + path = relative_project_path(project, raw, event.get("cwd")) + recall = ( + f"memory_recall_file(relative_file_path={json.dumps(path)})" + if path + else "memory_recall_file" + ) + text = ( + f"Daem0n: before this edit, call {recall} for past decisions and " + "warnings, and memory_preflight for the exact change." + ) + return json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": text, + } } - if pending is not None: - data = _data( - client.call( - "/v1/receipts/consume", - { - "workspace_id": workspace_id, - "host_session_id": host.host_session_id, - "edit_request_id": pending.edit_request_id, - "edit": wire_edit, - }, - ) - ) - if data.get("allowed") is not True: - raise RuntimeError - store.mark_consumed( - workspace_id=workspace_id, - native_session_id=native_session_id, - native_request_id=native_request_id, - credential_id=config.identity.credential_id, - edit_request_id=pending.edit_request_id, - ) - return PreEditResult(True, "") - data = _data( - client.call( - "/v1/edits", - { - "workspace_id": workspace_id, - "host_session_id": host.host_session_id, - "edit": wire_edit, - }, - ) - ) - store.save_pending( - native_session_id=native_session_id, - native_request_id=native_request_id, - credential_id=config.identity.credential_id, - pending=PendingEditBinding( - workspace_id, - host.host_session_id, - str(data["edit_request_id"]), - edit.edit_hash, - _expires(data["expires_at"]), - ), - ) - remedy = data.get("remedy") - return PreEditResult( - False, - str(remedy) if isinstance(remedy, Mapping) else "EDIT_BRIDGE_UNAVAILABLE", - ) - except Exception: - return PreEditResult(False, "EDIT_BRIDGE_UNAVAILABLE") + ) def main() -> None: - event = read_hook_event() - path = event.get("cwd") - if not isinstance(path, str) or not path: - block("EDIT_BRIDGE_UNAVAILABLE") - result = handle_pre_edit(event, path) - if not result.allowed: - block(result.message) + try: + output = reminder(read_hook_event()) + except Exception: + output = None + if output: + print(output) sys.exit(0) diff --git a/daem0nmcp/claude_hooks/stop.py b/daem0nmcp/claude_hooks/stop.py index 59766f3..609836f 100644 --- a/daem0nmcp/claude_hooks/stop.py +++ b/daem0nmcp/claude_hooks/stop.py @@ -1,20 +1,27 @@ -"""Claude Code Stop hook for fail-closed v7 memory reminders. +"""Claude Code Stop hook: remind about v7 memory calls, never block. -The hook analyzes the transcript but never writes memory directly. It emits -scoped, replay-safe ``memory_store`` and ``memory_record_outcome`` suggestions -that the authenticated MCP host can execute. +The hook reads the stdin event (``cwd``, ``transcript_path``, ``session_id``), +analyzes the transcript, and never writes memory. Its replay-safe +``memory_store`` and ``memory_record_outcome`` suggestions are shown to the +user as a ``systemMessage``; it never emits ``decision`` and always exits 0. """ import contextlib import hashlib import json -import os import re import sys +import time +from collections import deque from pathlib import Path from ..workspace import WorkspaceRegistry -from ._client import get_project_path, run_async, succeed +from ._client import ( + find_project_root, + read_hook_event, + relative_project_path, + run_async, +) # ─── transcript analysis ─────────────────────────────────────────── @@ -86,14 +93,13 @@ def _state_dir() -> Path: return d -def _state_file() -> Path: - session_id = os.environ.get("CLAUDE_SESSION_ID", "default") +def _state_file(session_id: str) -> Path: safe = re.sub(r"[^\w\-]", "_", session_id) return _state_dir() / f"stop_{safe}.json" -def _load_state() -> dict: - f = _state_file() +def _load_state(session_id: str) -> dict: + f = _state_file(session_id) if f.exists(): try: return json.loads(f.read_text(encoding="utf-8")) @@ -102,31 +108,56 @@ def _load_state() -> dict: return {"reminder_count": 0, "last_reminder_turn": -1} -def _save_state(state: dict) -> None: +_STATE_MAX_AGE_SECONDS = 7 * 24 * 3600 + + +def _save_state(session_id: str, state: dict) -> None: with contextlib.suppress(OSError): - _state_file().write_text(json.dumps(state), encoding="utf-8") + target = _state_file(session_id) + target.write_text(json.dumps(state), encoding="utf-8") + # One file per session: drop the ones no session has touched in a week. + cutoff = time.time() - _STATE_MAX_AGE_SECONDS + for old in target.parent.glob("stop_*.json"): + with contextlib.suppress(OSError): + if old.stat().st_mtime < cutoff: + old.unlink() # ─── transcript reading ─────────────────────────────────────────── -def _read_transcript() -> list[dict]: - path = os.environ.get("CLAUDE_TRANSCRIPT_PATH", "") - if not path or not Path(path).exists(): - return [] - messages = [] +_TRANSCRIPT_TAIL_RECORDS = 50 + + +def _read_transcript(path: object) -> tuple[list[dict], int]: + """Return the last Claude Code JSONL records and the total record count. + + Only the tail is JSON-parsed (the analysis looks at the last 10 records); + nested ``message`` objects are unwrapped. + """ + if not isinstance(path, str) or not path or not Path(path).is_file(): + return [], 0 + tail: deque[str] = deque(maxlen=_TRANSCRIPT_TAIL_RECORDS) + total = 0 try: - with open(path, encoding="utf-8") as f: + with open(path, encoding="utf-8", errors="replace") as f: for line in f: - line = line.strip() - if line: - try: - messages.append(json.loads(line)) - except json.JSONDecodeError: - continue + if line.strip(): + total += 1 + tail.append(line) except OSError: - pass - return messages + return [], 0 + messages = [] + for line in tail: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("message"), dict): + record = record["message"] + if isinstance(record, dict): + messages.append(record) + return messages, total def _get_recent_assistant_content(messages: list[dict], lookback: int = 5) -> str: @@ -216,14 +247,7 @@ def _workspace_id(project_path: str) -> str: def _relative_record_path(project_path: str, mentioned_path: str | None) -> str | None: if not mentioned_path: return None - try: - root = Path(project_path).resolve(strict=True) - candidate = Path(mentioned_path) - if not candidate.is_absolute(): - candidate = root / candidate - return candidate.resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None + return relative_project_path(Path(project_path), mentioned_path) def _memory_store_suggestion( @@ -287,6 +311,7 @@ async def analyse_and_remember( project_path: str, messages: list[dict], state: dict, + turn: int | None = None, ) -> StopResult: """ Core logic extracted for in-process testing. @@ -296,7 +321,7 @@ async def analyse_and_remember( * Returns result message (empty = nothing to say) * Updates *state* in-place for anti-loop tracking """ - current_turn = len(messages) + current_turn = len(messages) if turn is None else turn # Anti-loop check reminded_recently = state.get("last_reminder_turn", -1) >= current_turn - 2 @@ -331,10 +356,11 @@ async def analyse_and_remember( ) return StopResult( message=( - "[Daem0n suggests] Completion detected. The hook did not write " - "memory. Review each extracted decision, then execute:\n" + "Daem0n: this task looks finished and no outcome was recorded. " + "The hook wrote nothing. To keep these decisions, ask Claude " + "to run:\n" f"{suggestions}\n" - "When results are known, call " + "Once the result is known, ask Claude to call " "mcp__daem0nmcp__memory_record_outcome(" f'workspace_id="{workspace_id}", record_id="", ' 'outcome_text="", worked=true, ' @@ -344,10 +370,10 @@ async def analyse_and_remember( return StopResult( message=( - "[Daem0n whispers] Task completion detected. " - "If you made a durable decision, use memory_preflight for the exact " - "memory_store arguments before writing it. When a stored result is " - "known, call mcp__daem0nmcp__memory_record_outcome(" + "Daem0n: this task looks finished and no outcome was recorded. " + "The hook wrote nothing. To keep a durable decision, ask Claude to " + "run memory_preflight and then memory_store for it. Once the result " + "is known, ask Claude to call mcp__daem0nmcp__memory_record_outcome(" f'workspace_id="{workspace_id}", record_id="", ' 'outcome_text="", worked=true, ' f'idempotency_key="{outcome_key}").' @@ -359,20 +385,27 @@ async def analyse_and_remember( def main() -> None: - project_path = get_project_path() - if project_path is None: + event = read_hook_event() + project = find_project_root(event) + if project is None: sys.exit(0) + project_path = str(project) - messages = _read_transcript() + messages, turn = _read_transcript(event.get("transcript_path")) if not messages: sys.exit(0) - state = _load_state() - result = run_async(analyse_and_remember(project_path, messages, state)) - _save_state(state) + session_id = event.get("session_id") + if not isinstance(session_id, str) or not session_id: + session_id = "default" + state = _load_state(session_id) + result = run_async(analyse_and_remember(project_path, messages, state, turn)) + _save_state(session_id, state) if result.message: - succeed(result.message) + # systemMessage is shown to the user; the hook never returns a + # "decision", so it cannot keep the agent running. + print(json.dumps({"systemMessage": result.message})) sys.exit(0) diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..08aae92 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -16,7 +16,7 @@ python -m daem0nmcp.cli install-hooks [--force] python -m daem0nmcp.cli uninstall-hooks python -m daem0nmcp.cli install-claude-hooks [--dry-run] - python -m daem0nmcp.cli uninstall-claude-hooks [--dry-run] + python -m daem0nmcp.cli [--project-path P] uninstall-claude-hooks [--dry-run] [--remove-credentials] python -m daem0nmcp.cli install-opencode [--dry-run] [--force] python -m daem0nmcp.cli watch [--debounce SECONDS] [--no-system] [--no-log] [--no-poll] python -m daem0nmcp.cli index [--path PATH] [--patterns *.py *.ts ...] @@ -470,6 +470,16 @@ def main(): uninstall_claude_parser.add_argument( "--dry-run", action="store_true", help="Show what would change" ) + uninstall_claude_parser.add_argument( + "--remove-credentials", + action="store_true", + help="Also delete the project's local edit-bridge credential", + ) + for claude_parser in (install_claude_parser, uninstall_claude_parser): + # Also accept --project-path after the subcommand (default: cwd). + claude_parser.add_argument( + "--project-path", default=argparse.SUPPRESS, help="Project root path" + ) # install-opencode command install_oc_parser = subparsers.add_parser( @@ -929,7 +939,9 @@ def main(): from .claude_hooks.install import uninstall_claude_hooks success, message = uninstall_claude_hooks( - dry_run=getattr(args, "dry_run", False) + dry_run=getattr(args, "dry_run", False), + project_path=args.project_path or os.getcwd(), + remove_credentials=args.remove_credentials, ) if args.json: print(json.dumps({"success": success, "message": message})) diff --git a/daem0nmcp/edit_host.py b/daem0nmcp/edit_host.py index 2bb3d08..1f834e6 100644 --- a/daem0nmcp/edit_host.py +++ b/daem0nmcp/edit_host.py @@ -590,14 +590,11 @@ def provision_client_bridge_installation( ) -def provision_local_bridge_installation( - project_root: str | Path, - *, - config_root: Path | None = None, -) -> LocalBridgeInstallation: - """Provision or validate the local host credential outside the workspace.""" +def local_bridge_layout( + root: Path, config_root: Path | None +) -> tuple[Path, str, Path, Path]: + """Return storage path, principal, credential dir and socket dir for *root*.""" - root = Path(project_root).resolve(strict=True) settings = Settings(project_root=str(root)) storage_path = Path(settings.get_storage_path()).resolve(strict=False) principal_id = local_authority_principal(storage_path) @@ -608,12 +605,30 @@ def provision_local_bridge_installation( else Path.home() / ".daem0nmcp" / "edit-bridges" ) reject_linked_ancestry(base) - authority_directory = ensure_owner_only_directory(base.absolute() / authority_id) - credential_path = authority_directory / "credential.json" # AF_UNIX socket paths are limited to 104-108 bytes, and the authority # directory name alone is 64 characters, so sockets live in a short # sibling directory instead. - runtime_directory = base.absolute() / "run" / authority_id[:16] + return ( + storage_path, + principal_id, + base.absolute() / authority_id, + base.absolute() / "run" / authority_id[:16], + ) + + +def provision_local_bridge_installation( + project_root: str | Path, + *, + config_root: Path | None = None, +) -> LocalBridgeInstallation: + """Provision or validate the local host credential outside the workspace.""" + + root = Path(project_root).resolve(strict=True) + storage_path, principal_id, authority, runtime_directory = local_bridge_layout( + root, config_root + ) + authority_directory = ensure_owner_only_directory(authority) + credential_path = authority_directory / "credential.json" created = False if credential_path.exists(): _, identity = load_bridge_credential(credential_path) diff --git a/docs/v7-operations.md b/docs/v7-operations.md index ba19798..c7e8b8e 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -257,6 +257,35 @@ Install Claude hooks for one project: python -m daem0nmcp.cli --project-path /srv/project-a install-claude-hooks ``` +The hooks go into the user-level `~/.claude/settings.json` and run in every +project, staying silent unless the session directory, `CLAUDE_PROJECT_DIR`, or +one of their parents (not your home directory) contains `.daem0nmcp/`. They +remind only: + +| Event (matcher) | Hook | What it does | Input | Reminds only | +|-----------------|------|--------------|-------|--------------| +| `SessionStart` | `session_start` | Adds a line to the model's context asking it to call `session_brief` with this workspace's `workspace_id` | `CLAUDE_PROJECT_DIR` env | Yes | +| `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `memory_recall_file` for the file and `memory_preflight` for the change; silent elsewhere | stdin event | Yes | +| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | +| `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, shows you suggested `memory_preflight`/`memory_store`/`memory_record_outcome` calls to ask Claude for, as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | + +No hook blocks a tool call or keeps the agent running: every hook exits 0 and none +returns a `permissionDecision` or `decision`. + +Pairing writes an owner-only credential to +`~/.daem0nmcp/edit-bridges//credential.json`, uses +`~/.daem0nmcp/edit-bridges/run//` for the bridge socket, and +adds `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, +`DAEM0NMCP_PROJECT_ROOT` and `DAEM0NMCP_STORAGE_PATH` to +`/.claude/settings.local.json`. Remove the hooks, those env keys and +(with `--remove-credentials`) the credential and socket directories with: + +```bash +python -m daem0nmcp.cli uninstall-claude-hooks --project-path /srv/project-a --remove-credentials +``` + Install OpenCode V1 locally: ```bash diff --git a/hooks/daem0n_post_edit_hook.py b/hooks/daem0n_post_edit_hook.py index 0a4397f..fb431b5 100644 --- a/hooks/daem0n_post_edit_hook.py +++ b/hooks/daem0n_post_edit_hook.py @@ -1,141 +1,11 @@ #!/usr/bin/env python3 -"""Read-only Claude PostToolUse guidance for significant v7 changes.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os import sys -from pathlib import Path - -SIGNIFICANT_EXTENSIONS = frozenset( - { - ".py", - ".ts", - ".js", - ".go", - ".rs", - ".java", - ".yaml", - ".yml", - ".json", - ".toml", - ".sql", - } -) -SIGNIFICANT_TERMS = ( - "async def ", - "class ", - "auth", - "config", - "credential", - "endpoint", - "migration", - "schema", - "secret", - "token", +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, ) - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _tool_input() -> dict[str, object]: - try: - value = json.loads(os.environ.get("TOOL_INPUT", "{}")) - except (json.JSONDecodeError, TypeError): - return {} - return value if isinstance(value, dict) else {} - - -def _relative_path(root: Path, value: str) -> str | None: - try: - return Path(value).resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None - - -def _is_significant(value: str, changed: str) -> bool: - if Path(value).suffix.casefold() not in SIGNIFICANT_EXTENSIONS: - return False - lowered = changed.casefold() - return len(changed) > 500 or any(term in lowered for term in SIGNIFICANT_TERMS) - - -def _suggestion(root: Path, value: str, changed: str) -> str: - key = os.path.normcase(str(root)) - workspace_id = "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - relative_path = _relative_path(root, value) - path_label = relative_path or Path(value).name or "changed file" - digest = hashlib.sha256( - f"{workspace_id}\0{path_label}\0{changed}".encode("utf-8") - ).hexdigest()[:24] - target_arguments: dict[str, object] = { - "record_type": "decision", - "content": f"Summarize the significant change to {path_label}", - "rationale": "Preserve the reason and constraints for future sessions", - "idempotency_key": f"hook-postedit-{digest}", - } - if relative_path is not None: - target_arguments["relative_file_path"] = relative_path - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - store_fields = [ - f'workspace_id="{workspace_id}"', - 'record_type="decision"', - f'content={json.dumps(target_arguments["content"])}', - f'rationale={json.dumps(target_arguments["rationale"])}', - ] - if relative_path is not None: - store_fields.append(f"relative_file_path={json.dumps(relative_path)}") - store_fields.extend( - ( - f'idempotency_key="{target_arguments["idempotency_key"]}"', - 'preflight_token=""', - ) - ) - return ( - "[Daem0n suggests] This hook did not write memory. First call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - f'description="Record the significant change to {path_label}"). Then ' - "call mcp__daem0nmcp__memory_store(" + ", ".join(store_fields) + ")." - ) - - -def main() -> None: - root = _configured_root() - if root is None: - return - data = _tool_input() - value = data.get("file_path") - if not isinstance(value, str) or not value: - return - changed = " ".join( - item - for item in ( - data.get("old_string"), - data.get("new_string"), - data.get("content"), - ) - if isinstance(item, str) - ) - if _is_significant(value, changed): - print(_suggestion(root, value, changed)) - - -if __name__ == "__main__": - try: - main() - except Exception: - print( - "[Daem0n v7 warning] No memory was written. Call memory_preflight " - "before any memory_store retry, and use system_health for diagnostics." - ) - sys.exit(0) +sys.exit(0) diff --git a/hooks/daem0n_pre_edit_hook.py b/hooks/daem0n_pre_edit_hook.py index 5dc0a56..fb431b5 100644 --- a/hooks/daem0n_pre_edit_hook.py +++ b/hooks/daem0n_pre_edit_hook.py @@ -1,99 +1,11 @@ #!/usr/bin/env python3 -"""Fail-closed Claude PreToolUse guidance for v7-scoped edits.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os import sys -from pathlib import Path - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _tool_path() -> str | None: - try: - data = json.loads(os.environ.get("TOOL_INPUT", "{}")) - except (json.JSONDecodeError, TypeError): - return None - if not isinstance(data, dict): - return None - value = data.get("file_path") or data.get("notebook_path") - return value if isinstance(value, str) and value else None - - -def _relative_path(root: Path, value: str) -> str | None: - try: - return Path(value).resolve(strict=False).relative_to(root).as_posix() - except (OSError, RuntimeError, ValueError): - return None - - -def _workspace_id(root: Path) -> str: - key = os.path.normcase(str(root)) - return "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - - -def _guidance(root: Path, value: str) -> str: - workspace_id = _workspace_id(root) - relative_path = _relative_path(root, value) - path_label = relative_path or "" - digest = hashlib.sha256( - f"{workspace_id}\0{path_label}".encode("utf-8") - ).hexdigest()[:24] - target_arguments = { - "record_type": "decision", - "content": f"Describe the planned change to {path_label}", - "idempotency_key": f"hook-preedit-{digest}", - } - if relative_path is not None: - target_arguments["relative_file_path"] = relative_path - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - return ( - "[Daem0n blocks] This standalone hook cannot access the authenticated " - "v7 invocation, so it fails closed. Ask the MCP host to call " - f'mcp__daem0nmcp__memory_recall(workspace_id="{workspace_id}", ' - f'query="planned edit to {path_label}", limit=10). If the plan is ' - "durable, call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - f'description="Review the planned edit to {path_label}"). Review the ' - "returned warnings and use its preflight_token only with that exact " - "memory_store request." - ) - - -def main() -> int: - root = _configured_root() - if root is None: - return 0 - value = _tool_path() - if value is None: - print( - "[Daem0n blocks] Unable to identify the edit target; this hook " - "fails closed. Call memory_recall and memory_preflight through the " - "authenticated MCP host.", - file=sys.stderr, - ) - return 2 - try: - message = _guidance(root, value) - except Exception: - message = ( - "[Daem0n blocks] Unable to construct scoped v7 guidance; this " - "hook fails closed. Call session_brief, memory_recall, and " - "memory_preflight through the authenticated MCP host." - ) - print(message, file=sys.stderr) - return 2 - -if __name__ == "__main__": - sys.exit(main()) +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, +) +sys.exit(0) diff --git a/hooks/daem0n_prompt_hook.py b/hooks/daem0n_prompt_hook.py index 9b1f2ea..fb431b5 100755 --- a/hooks/daem0n_prompt_hook.py +++ b/hooks/daem0n_prompt_hook.py @@ -1,49 +1,11 @@ #!/usr/bin/env python3 -"""Read-only Claude UserPromptSubmit guidance for the v7 ritual.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import os import sys -from pathlib import Path - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _workspace_id(root: Path) -> str: - key = os.path.normcase(str(root)) - return "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - - -def main() -> None: - """Tell the authenticated host which bounded read calls to make.""" - root = _configured_root() - if root is None: - return - workspace_id = _workspace_id(root) - print( - "[Daem0n v7 reminder] At session start call " - f'mcp__daem0nmcp__session_brief(workspace_id="{workspace_id}"). ' - "When history is relevant, call " - f'mcp__daem0nmcp__memory_recall(workspace_id="{workspace_id}", ' - 'query="", limit=10). Use only the opaque workspace_id ' - "issued for this workspace." - ) - -if __name__ == "__main__": - try: - main() - except Exception: - print( - "[Daem0n v7 reminder] Call session_brief before protected work; " - "use system_health if the scoped briefing cannot be obtained." - ) - sys.exit(0) +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, +) +sys.exit(0) diff --git a/hooks/daem0n_stop_hook.py b/hooks/daem0n_stop_hook.py index db8f91c..fb431b5 100755 --- a/hooks/daem0n_stop_hook.py +++ b/hooks/daem0n_stop_hook.py @@ -1,149 +1,11 @@ #!/usr/bin/env python3 -"""Stateless, read-only Claude Stop guidance for the v7 outcome ritual.""" +"""Deprecated manual Claude Code hook; kept so old settings entries exit 0.""" -from __future__ import annotations - -import hashlib -import json -import os -import re import sys -from pathlib import Path - -_MAX_TRANSCRIPT_BYTES = 1_000_000 -_MAX_MESSAGES = 50 -_COMPLETION = re.compile( - r"(?i)\b(?:implementation|task|work|feature)\s+(?:is\s+)?" - r"(?:complete|done|finished)\b|\ball tasks?\s+(?:are\s+)?done\b" -) -_DECISION = re.compile( - r"(?i)\bi (?:will|chose|decided to)\s+(.{20,240}?)(?:\.|$)" -) -_OUTCOME_TOOL_NAMES = frozenset( - { - "memory_record_outcome", - "daem0nmcp_memory_record_outcome", - "mcp__daem0nmcp__memory_record_outcome", - } +print( + "Daem0n: this manual hook is deprecated; " + "run python -m daem0nmcp.cli install-claude-hooks", + file=sys.stderr, ) - - -def _configured_root() -> Path | None: - raw_root = os.environ.get("CLAUDE_PROJECT_DIR", "") - if not raw_root: - return None - root = Path(raw_root).resolve(strict=False) - return root if (root / ".daem0nmcp").is_dir() else None - - -def _messages() -> list[dict[str, object]]: - raw_path = os.environ.get("CLAUDE_TRANSCRIPT_PATH", "") - if not raw_path: - return [] - path = Path(raw_path) - try: - if not path.is_file() or path.stat().st_size > _MAX_TRANSCRIPT_BYTES: - return [] - lines = path.read_text(encoding="utf-8").splitlines()[-_MAX_MESSAGES:] - except OSError: - return [] - messages: list[dict[str, object]] = [] - for line in lines: - try: - value = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(value, dict): - messages.append(value) - return messages - - -def _recent_content(messages: list[dict[str, object]]) -> tuple[str, set[str]]: - text_parts: list[str] = [] - tools: set[str] = set() - for message in messages: - if message.get("role") != "assistant": - continue - content = message.get("content") - if isinstance(content, str): - text_parts.append(content) - elif isinstance(content, list): - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") == "text" and isinstance(part.get("text"), str): - text_parts.append(str(part["text"])) - if part.get("type") == "tool_use" and isinstance(part.get("name"), str): - tools.add(str(part["name"]).casefold()) - return " ".join(text_parts)[-20_000:], tools - - -def _reason(root: Path, content: str) -> str: - key = os.path.normcase(str(root)) - workspace_id = "ws_" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:24] - match = _DECISION.search(content) - decision = ( - re.sub(r"\s+", " ", match.group(1)).strip() - if match is not None - else "Summarize the durable decision from the completed task" - ) - digest = hashlib.sha256( - f"{workspace_id}\0{decision}".encode("utf-8") - ).hexdigest()[:24] - store_key = f"hook-stop-{digest}" - outcome_key = f"hook-outcome-{digest}" - target_arguments = { - "record_type": "decision", - "content": decision, - "rationale": "Preserve the verified decision from the completed task", - "idempotency_key": store_key, - } - encoded = json.dumps(target_arguments, ensure_ascii=True, separators=(",", ":")) - return ( - "[Daem0n blocks] Completion was detected and this hook did not mutate " - "memory. Review and call " - f'mcp__daem0nmcp__memory_preflight(workspace_id="{workspace_id}", ' - f'target_tool="memory_store", target_arguments={encoded}, ' - 'description="Record the completed task decision"); then call ' - f'mcp__daem0nmcp__memory_store(workspace_id="{workspace_id}", ' - f'record_type="decision", content={json.dumps(decision)}, ' - 'rationale="Preserve the verified decision from the completed task", ' - f'idempotency_key="{store_key}", ' - 'preflight_token=""). Once verified, call ' - f'mcp__daem0nmcp__memory_record_outcome(workspace_id="{workspace_id}", ' - 'record_id="", outcome_text="", worked=true, ' - f'idempotency_key="{outcome_key}").' - ) - - -def main() -> None: - root = _configured_root() - if root is None: - return - content, tools = _recent_content(_messages()) - if not content or not _COMPLETION.search(content): - return - if tools.intersection(_OUTCOME_TOOL_NAMES): - return - print(json.dumps({"decision": "block", "reason": _reason(root, content)})) - - -if __name__ == "__main__": - try: - main() - except Exception: - print( - json.dumps( - { - "decision": "block", - "reason": ( - "[Daem0n blocks] Unable to verify v7 outcome state. " - "Call session_brief and system_health, then use " - "memory_preflight, memory_store, and " - "memory_record_outcome through the authenticated host." - ), - } - ) - ) - sys.exit(0) +sys.exit(0) diff --git a/hooks/settings.json.example b/hooks/settings.json.example deleted file mode 100644 index 7cc7d38..0000000 --- a/hooks/settings.json.example +++ /dev/null @@ -1,70 +0,0 @@ -{ - "hooks": { - "SessionStart": [ - { - "matcher": "startup|resume", - "hooks": [ - { - "type": "command", - "command": "echo '[Daem0n v7] Before responding, call mcp__daem0nmcp__session_brief(workspace_id=\"\").'" - } - ] - } - ], - "UserPromptSubmit": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_prompt_hook.py\"" - } - ] - } - ], - "PreToolUse": [ - { - "matcher": "Edit|Write|NotebookEdit", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_pre_edit_hook.py\"" - } - ] - } - ], - "PostToolUse": [ - { - "matcher": "Edit|Write", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_post_edit_hook.py\"" - } - ] - } - ], - "Stop": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - } - ] - } - ], - "SubagentStop": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "python3 \"$HOME/Daem0nMCP/hooks/daem0n_stop_hook.py\"" - } - ] - } - ] - } -} diff --git a/tests/api_v7/test_docs_hooks.py b/tests/api_v7/test_docs_hooks.py index 0fdc00e..3fef514 100644 --- a/tests/api_v7/test_docs_hooks.py +++ b/tests/api_v7/test_docs_hooks.py @@ -10,6 +10,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -35,7 +36,6 @@ ROOT / "hooks" / "daem0n_pre_edit_hook.py", ROOT / "hooks" / "daem0n_post_edit_hook.py", ROOT / "hooks" / "daem0n_stop_hook.py", - ROOT / "hooks" / "settings.json.example", ) ROOT_HOOKS = tuple(path for path in CUTOVER_FILES if path.parent == ROOT / "hooks") @@ -100,7 +100,7 @@ def _run_root_hook( name: str, *, workspace_root: Path, - extra_environment: dict[str, str] | None = None, + event: dict[str, object], ) -> subprocess.CompletedProcess[str]: environment = os.environ.copy() environment.update( @@ -111,12 +111,11 @@ def _run_root_hook( "PYTHONDONTWRITEBYTECODE": "1", } ) - if extra_environment: - environment.update(extra_environment) return subprocess.run( [sys.executable, str(ROOT / "hooks" / name)], - cwd=ROOT, + cwd=workspace_root, env=environment, + input=json.dumps(event), capture_output=True, text=True, timeout=5, @@ -221,42 +220,15 @@ def test_hooks_do_not_open_the_mutable_v6_manager_path(self) -> None: self.assertNotIn("memory.remember", hook_sources) self.assertNotIn("INSERT INTO session_state", hook_sources) - def test_root_hooks_are_read_only_v7_host_guidance(self) -> None: - hook_sources = "\n".join(_read(path) for path in ROOT_HOOKS) - - for forbidden in ( - "subprocess", - "daem0nmcp.cli", - "os.system", - "os.popen", - "urllib.request", - "sqlite3", - ".write_text(", - ".write_bytes(", - ".mkdir(", - ".touch(", - ".unlink(", - "open(", - ): - with self.subTest(forbidden=forbidden): - self.assertNotIn(forbidden, hook_sources) - - expected_calls = { - "daem0n_prompt_hook.py": ("session_brief", "memory_recall"), - "daem0n_pre_edit_hook.py": ("memory_recall", "memory_preflight"), - "daem0n_post_edit_hook.py": ("memory_preflight", "memory_store"), - "daem0n_stop_hook.py": ( - "memory_preflight", - "memory_store", - "memory_record_outcome", - ), - "settings.json.example": ("session_brief",), - } + def test_root_hooks_are_stdlib_only_deprecation_stubs(self) -> None: for path in ROOT_HOOKS: text = _read(path) - for tool_name in expected_calls[path.name]: - with self.subTest(path=path.name, tool_name=tool_name): - self.assertIn(tool_name, text) + with self.subTest(path=path.name): + imports = re.findall(r"^\s*(?:import|from)\s+(\S+)", text, re.M) + self.assertEqual(["sys"], imports) + self.assertIn("install-claude-hooks", text) + self.assertIn("sys.exit(0)", text) + self.assertNotIn("exit(2)", text) class HookNameTests(unittest.TestCase): @@ -321,103 +293,62 @@ async def test_stop_suggests_replay_safe_v7_calls_without_writing(self) -> None: self.assertIn("preflight_token", result.message) self.assertFalse((project / ".daem0nmcp" / "storage").exists()) - async def test_pre_edit_fails_closed_without_native_bridge(self) -> None: - from daem0nmcp.claude_hooks.pre_edit import async_main + def test_pre_edit_reminds_without_blocking(self) -> None: + from daem0nmcp.claude_hooks.pre_edit import reminder - with tempfile.TemporaryDirectory() as tmp_dir: - project = Path(tmp_dir) + with ( + tempfile.TemporaryDirectory() as tmp_dir, + # Bound the upward project search to the temp dir. + mock.patch.object(Path, "home", return_value=Path(tmp_dir)), + mock.patch.dict(os.environ, {"CLAUDE_PROJECT_DIR": ""}), + ): + project = Path(tmp_dir) / "project" + project.mkdir() + event = { + "cwd": str(project), + "tool_name": "Edit", + "tool_input": {"file_path": str(project / "server.py")}, + } + self.assertIsNone(reminder(event)) (project / ".daem0nmcp").mkdir() - result = await async_main(str(project), str(project / "server.py")) + output = json.loads(reminder(event) or "{}")["hookSpecificOutput"] - self.assertFalse(result.allowed) - self.assertEqual("EDIT_BRIDGE_UNAVAILABLE", result.message) + self.assertNotIn("permissionDecision", output) + self.assertIn("memory_preflight", output["additionalContext"]) + self.assertIn( + 'memory_recall_file(relative_file_path="server.py")', + output["additionalContext"], + ) self.assertFalse((project / ".daem0nmcp" / "storage").exists()) class RootHookProcessTests(unittest.TestCase): - def test_pre_edit_process_denies_without_mutating_workspace(self) -> None: + def test_legacy_stubs_exit_zero_on_a_pre_tool_use_payload(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: workspace_root = Path(tmp_dir) (workspace_root / ".daem0nmcp").mkdir() before = _tree_snapshot(workspace_root) - - result = _run_root_hook( - "daem0n_pre_edit_hook.py", - workspace_root=workspace_root, - extra_environment={ - "TOOL_INPUT": json.dumps( - {"file_path": str(workspace_root / "service.py")} - ) - }, - ) - - self.assertEqual(2, result.returncode) - self.assertEqual("", result.stdout) - self.assertIn("fails closed", result.stderr) - self.assertIn("memory_recall", result.stderr) - self.assertIn("memory_preflight", result.stderr) - self.assertRegex(result.stderr, r"workspace_id=[\"']ws_[a-f0-9]{24}") - self.assertIn("target_tool", result.stderr) - self.assertIn("target_arguments", result.stderr) - self.assertEqual(before, _tree_snapshot(workspace_root)) - - def test_advisory_processes_emit_exact_v7_calls_without_writing(self) -> None: - with tempfile.TemporaryDirectory() as tmp_dir: - workspace_root = Path(tmp_dir) - (workspace_root / ".daem0nmcp").mkdir() - transcript = workspace_root / "transcript.jsonl" - transcript.write_text( - json.dumps( - { - "role": "assistant", - "content": ( - "I will use signed cookies because they avoid shared " - "server state. Implementation is complete." - ), - } - ) - + "\n", - encoding="utf-8", - ) - before = _tree_snapshot(workspace_root) - tool_input = json.dumps( - { + event = { + "session_id": "s-1", + "cwd": str(workspace_root), + "hook_event_name": "PreToolUse", + "tool_name": "Edit", + "tool_input": { "file_path": str(workspace_root / "service.py"), - "new_string": "async def authenticate(token): pass", - } - ) - - prompt = _run_root_hook( - "daem0n_prompt_hook.py", - workspace_root=workspace_root, - ) - post_edit = _run_root_hook( - "daem0n_post_edit_hook.py", - workspace_root=workspace_root, - extra_environment={"TOOL_INPUT": tool_input}, - ) - stop = _run_root_hook( - "daem0n_stop_hook.py", - workspace_root=workspace_root, - extra_environment={ - "CLAUDE_TRANSCRIPT_PATH": str(transcript), + "old_string": "a", + "new_string": "b", }, - ) + } - self.assertEqual(0, prompt.returncode) - self.assertIn("session_brief", prompt.stdout) - self.assertIn("memory_recall", prompt.stdout) - self.assertEqual(0, post_edit.returncode) - self.assertIn("memory_preflight", post_edit.stdout) - self.assertIn("memory_store", post_edit.stdout) - self.assertIn("idempotency_key", post_edit.stdout) - self.assertEqual(0, stop.returncode) - stop_payload = json.loads(stop.stdout) - self.assertEqual("block", stop_payload["decision"]) - self.assertIn("memory_preflight", stop_payload["reason"]) - self.assertIn("memory_store", stop_payload["reason"]) - self.assertIn("memory_record_outcome", stop_payload["reason"]) - self.assertIn("idempotency_key", stop_payload["reason"]) + for path in ROOT_HOOKS: + with self.subTest(path=path.name): + result = _run_root_hook( + path.name, workspace_root=workspace_root, event=event + ) + self.assertEqual(0, result.returncode) + self.assertEqual("", result.stdout) + self.assertIn("deprecated", result.stderr) + self.assertIn("install-claude-hooks", result.stderr) self.assertEqual(before, _tree_snapshot(workspace_root)) diff --git a/tests/api_v7/test_process_edit_capture.py b/tests/api_v7/test_process_edit_capture.py index 6118ab9..b226d46 100644 --- a/tests/api_v7/test_process_edit_capture.py +++ b/tests/api_v7/test_process_edit_capture.py @@ -12,7 +12,6 @@ from daem0nmcp.claude_hooks.post_edit_preflight import ( handle_edit_preflight_response, ) -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.database import DatabaseManager from daem0nmcp.edit_host import ( EditHostConfig, @@ -21,6 +20,7 @@ ) from daem0nmcp.workspace import WorkspaceRegistry from tests.api_v7.process_client import call, process_client, succeed +from tests.native_edit_host import drive_native_edit @pytest.fixture @@ -68,8 +68,8 @@ async def test_actual_stdio_mcp_shares_bridge_authority_and_promotes_capture( ) as session: await succeed(session, "session_brief", scope) with patch.dict(os.environ, environment, clear=False): - denied = handle_pre_edit(first_event, str(workspace.root)) - assert not denied.allowed + denied = drive_native_edit(first_event, str(workspace.root)) + assert not denied normalized = native_edit_request( project_path=workspace.root, tool_name="Edit", @@ -112,8 +112,8 @@ async def test_actual_stdio_mcp_shares_bridge_authority_and_promotes_capture( ) retry_event = {**first_event, "tool_use_id": "tool-use-allowed-2"} with patch.dict(os.environ, environment, clear=False): - allowed = handle_pre_edit(retry_event, str(workspace.root)) - assert allowed.allowed + allowed = drive_native_edit(retry_event, str(workspace.root)) + assert allowed target.write_text("after", encoding="utf-8") with patch.dict(os.environ, environment, clear=False): assert handle_post_edit(retry_event, str(workspace.root)) diff --git a/tests/native_edit_host.py b/tests/native_edit_host.py new file mode 100644 index 0000000..e820e12 --- /dev/null +++ b/tests/native_edit_host.py @@ -0,0 +1,121 @@ +"""Test-only native host driver for the edit bridge. + +The Claude ``pre_edit`` hook no longer drives the bridge (it only reminds), but +the bridge itself stays until its planned removal. These steps are what a +native host does before an edit: open a host session, then either consume an +approved receipt or create a pending edit request. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from daem0nmcp.claude_hooks.native_edit import ( + configured_native_edit_tools, + native_edit_request, +) +from daem0nmcp.edit_host import ( + CreatedHostSession, + EditHostConfig, + EditHostStateStore, + PendingEditBinding, +) + + +def _expires(value: object) -> datetime: + assert isinstance(value, str) + return datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + + +def _data(response: object) -> Mapping[str, Any]: + body = getattr(response, "body", None) + assert isinstance(body, Mapping) and body.get("ok") is True, body + return body["data"] + + +def drive_native_edit(event: Mapping[str, Any], project_path: str) -> bool: + """Return True when an approved receipt was consumed for this exact edit.""" + native_session_id = event["session_id"] + native_request_id = event["tool_use_id"] + config = EditHostConfig.from_environment(os.environ) + workspace_id = config.workspace_id(project_path) + store = EditHostStateStore(config) + client = config.build_client() + edit = native_edit_request( + project_path=project_path, + tool_name=event["tool_name"], + tool_input=event["tool_input"], + configured_tools=configured_native_edit_tools(os.environ), + ) + + def create() -> CreatedHostSession: + data = _data(client.call("/v1/sessions", {"workspace_id": workspace_id})) + return CreatedHostSession( + str(data["host_session_id"]), _expires(data["expires_at"]) + ) + + credential_id = config.identity.credential_id + host = store.load_or_create_session( + workspace_id=workspace_id, + native_session_id=native_session_id, + credential_id=credential_id, + create=create, + ) + pending = store.get_pending( + workspace_id=workspace_id, + native_session_id=native_session_id, + credential_id=credential_id, + edit_hash=edit.edit_hash, + ) + wire_edit = { + "tool_name": edit.tool_name, + "arguments": dict(edit.arguments), + "preimages": [item.canonical() for item in edit.preimages], + } + if pending is not None: + data = _data( + client.call( + "/v1/receipts/consume", + { + "workspace_id": workspace_id, + "host_session_id": host.host_session_id, + "edit_request_id": pending.edit_request_id, + "edit": wire_edit, + }, + ) + ) + assert data.get("allowed") is True, data + store.mark_consumed( + workspace_id=workspace_id, + native_session_id=native_session_id, + native_request_id=native_request_id, + credential_id=credential_id, + edit_request_id=pending.edit_request_id, + ) + return True + data = _data( + client.call( + "/v1/edits", + { + "workspace_id": workspace_id, + "host_session_id": host.host_session_id, + "edit": wire_edit, + }, + ) + ) + store.save_pending( + native_session_id=native_session_id, + native_request_id=native_request_id, + credential_id=credential_id, + pending=PendingEditBinding( + workspace_id, + host.host_session_id, + str(data["edit_request_id"]), + edit.edit_hash, + _expires(data["expires_at"]), + ), + ) + return False diff --git a/tests/test_claude_hooks_cli.py b/tests/test_claude_hooks_cli.py index 18ba48a..2eaf291 100644 --- a/tests/test_claude_hooks_cli.py +++ b/tests/test_claude_hooks_cli.py @@ -42,3 +42,82 @@ def test_cli_uninstall_claude_hooks_dry_run(fake_settings): ok, msg = uninstall_claude_hooks(dry_run=True) assert ok assert "No Daem0n hooks found" in msg + + +def _cli(monkeypatch, capsys, *argv: str) -> tuple[int, str]: + from daem0nmcp import cli + + monkeypatch.setattr("sys.argv", ["daem0nmcp.cli", *argv]) + with pytest.raises(SystemExit) as exc_info: + cli.main() + return exc_info.value.code, capsys.readouterr().out + + +def test_cli_uninstall_accepts_project_path_after_the_subcommand( + fake_settings, tmp_path, monkeypatch, capsys +): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("DAEM0NMCP_PROJECT_ROOT", "unset-by-test") + project = tmp_path / "project" + project.mkdir() + + code, _out = _cli( + monkeypatch, capsys, "--project-path", str(project), "install-claude-hooks" + ) + assert code == 0 + bridges = home / ".daem0nmcp" / "edit-bridges" + credential_dirs = [p for p in bridges.iterdir() if p.name != "run"] + assert len(credential_dirs) == 1 + + code, out = _cli( + monkeypatch, + capsys, + "uninstall-claude-hooks", + "--project-path", + str(project), + "--dry-run", + "--remove-credentials", + ) + assert code == 0 + assert f"[dry-run] Would remove {credential_dirs[0]}" in out + assert credential_dirs[0].exists() + + code, out = _cli( + monkeypatch, + capsys, + "uninstall-claude-hooks", + "--project-path", + str(project), + "--remove-credentials", + ) + assert code == 0, out + assert not credential_dirs[0].exists() + local = json.loads( + (project / ".claude" / "settings.local.json").read_text(encoding="utf-8") + ) + assert local == {} + + +def test_cli_remove_credentials_defaults_to_the_current_directory( + fake_settings, tmp_path, monkeypatch, capsys +): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + project = tmp_path / "project" + project.mkdir() + monkeypatch.chdir(project) + + assert _cli(monkeypatch, capsys, "install-claude-hooks")[0] == 0 + code, out = _cli( + monkeypatch, capsys, "uninstall-claude-hooks", "--remove-credentials" + ) + + assert code == 0, out + assert "edit-bridges" in out + bridges = home / ".daem0nmcp" / "edit-bridges" + assert [p for p in bridges.iterdir() if p.name != "run"] == [] diff --git a/tests/test_claude_hooks_client.py b/tests/test_claude_hooks_client.py index 356602b..40b7144 100644 --- a/tests/test_claude_hooks_client.py +++ b/tests/test_claude_hooks_client.py @@ -9,9 +9,7 @@ from daem0nmcp.claude_hooks._client import ( block, get_command_from_input, - get_file_path_from_input, get_project_path, - get_tool_input, read_hook_event, run_hook_safely, succeed, @@ -108,33 +106,17 @@ def test_rejects_oversized_or_non_object_stdin_event(self, monkeypatch): monkeypatch.setattr(sys, "stdin", TextIOWrapper(BytesIO(b"[]"))) assert read_hook_event() == {} - def test_file_path_from_edit(self, monkeypatch): - monkeypatch.setenv("TOOL_INPUT", json.dumps({"file_path": "/foo/bar.py"})) - assert get_file_path_from_input() == "/foo/bar.py" - - def test_notebook_path(self, monkeypatch): - monkeypatch.setenv("TOOL_INPUT", json.dumps({"notebook_path": "/foo.ipynb"})) - assert get_file_path_from_input() == "/foo.ipynb" - - def test_file_path_takes_precedence(self, monkeypatch): - monkeypatch.setenv( - "TOOL_INPUT", - json.dumps({"file_path": "/a.py", "notebook_path": "/b.ipynb"}), - ) - assert get_file_path_from_input() == "/a.py" - def test_command_from_bash(self, monkeypatch): monkeypatch.setenv("TOOL_INPUT", json.dumps({"command": "ls -la"})) assert get_command_from_input() == "ls -la" def test_invalid_json(self, monkeypatch): monkeypatch.setenv("TOOL_INPUT", "not json") - assert get_tool_input() == {} - assert get_file_path_from_input() is None + assert get_command_from_input() is None def test_missing_env(self, monkeypatch): monkeypatch.delenv("TOOL_INPUT", raising=False) - assert get_tool_input() == {} + assert get_command_from_input() is None class TestRunHookSafely: diff --git a/tests/test_claude_hooks_install.py b/tests/test_claude_hooks_install.py index 867e4f4..a44444e 100644 --- a/tests/test_claude_hooks_install.py +++ b/tests/test_claude_hooks_install.py @@ -1,6 +1,7 @@ """Tests for the Claude Code hook installer.""" import json +import shutil import ssl from pathlib import Path @@ -295,3 +296,208 @@ def test_dry_run_no_write(self, fake_settings): # File should be unchanged assert fake_settings.read_text() == original + + def test_removes_legacy_manual_hook_entries(self, fake_settings): + fake_settings.write_text( + json.dumps( + { + "hooks": { + name: [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": f'python3 "$HOME/Daem0nMCP/hooks/{script}"', + } + ], + } + ] + for name, script in ( + ("UserPromptSubmit", "daem0n_prompt_hook.py"), + ("PreToolUse", "daem0n_pre_edit_hook.py"), + ("PostToolUse", "daem0n_post_edit_hook.py"), + ("Stop", "daem0n_stop_hook.py"), + ) + } + } + ) + ) + + ok, _msg = uninstall_claude_hooks() + + assert ok + assert json.loads(fake_settings.read_text())["hooks"] == {} + + def test_project_cleanup_strips_pairing_env_and_credentials( + self, fake_settings, tmp_path + ): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + ok, message = install_claude_hooks( + project_path=project, bridge_config_root=host_config + ) + assert ok, message + local_settings = project / ".claude" / "settings.local.json" + value = json.loads(local_settings.read_text(encoding="utf-8")) + value["env"]["USER_KEY"] = "kept" + value["permissions"] = {"allow": ["Bash(ls)"]} + local_settings.write_text(json.dumps(value), encoding="utf-8") + credential = Path(value["env"]["DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE"]) + runtime = Path(value["env"]["DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR"]) + runtime.mkdir(parents=True) + + ok, message = uninstall_claude_hooks( + dry_run=True, + project_path=project, + remove_credentials=True, + bridge_config_root=host_config, + ) + assert ok, message + assert credential.is_file() and runtime.is_dir() + assert "DAEM0NMCP_PROJECT_ROOT" in local_settings.read_text(encoding="utf-8") + + ok, message = uninstall_claude_hooks( + project_path=project, + remove_credentials=True, + bridge_config_root=host_config, + ) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == { + "env": {"USER_KEY": "kept"}, + "permissions": {"allow": ["Bash(ls)"]}, + } + assert not credential.parent.exists() + assert not runtime.exists() + assert runtime.parent.name == "run" + + def test_project_cleanup_without_credentials_keeps_them( + self, fake_settings, tmp_path + ): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + install_claude_hooks(project_path=project, bridge_config_root=host_config) + local_settings = project / ".claude" / "settings.local.json" + credential = Path( + json.loads(local_settings.read_text(encoding="utf-8"))["env"][ + "DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE" + ] + ) + + ok, message = uninstall_claude_hooks(project_path=project) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == {} + assert credential.is_file() + + +def _paired_project(tmp_path): + project = tmp_path / "project" + project.mkdir() + host_config = tmp_path / "host-config" + ok, message = install_claude_hooks( + project_path=project, bridge_config_root=host_config + ) + assert ok, message + local_settings = project / ".claude" / "settings.local.json" + env = json.loads(local_settings.read_text(encoding="utf-8"))["env"] + credential_dir = Path(env["DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE"]).parent + runtime = Path(env["DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR"]) + runtime.mkdir(parents=True) + return project, host_config, local_settings, credential_dir, runtime + + +class TestUninstallRobustness: + def test_unpaired_project_keeps_general_daem0n_keys(self, fake_settings, tmp_path): + project = tmp_path / "project" + (project / ".claude").mkdir(parents=True) + local_settings = project / ".claude" / "settings.local.json" + original = json.dumps( + {"env": {"DAEM0NMCP_PROJECT_ROOT": "x", "DAEM0NMCP_STORAGE_PATH": "y"}} + ) + local_settings.write_text(original, encoding="utf-8") + + ok, _message = uninstall_claude_hooks(project_path=project) + + assert ok + assert local_settings.read_text(encoding="utf-8") == original + + def test_bom_prefixed_local_settings_are_cleaned(self, fake_settings, tmp_path): + project, _host, local_settings, _cred, _runtime = _paired_project(tmp_path) + text = local_settings.read_text(encoding="utf-8") + local_settings.write_bytes(b"\xef\xbb\xbf" + text.encode("utf-8")) + + ok, message = uninstall_claude_hooks(project_path=project) + + assert ok, message + assert json.loads(local_settings.read_text(encoding="utf-8")) == {} + + def test_malformed_local_settings_report_partial_success( + self, fake_settings, tmp_path + ): + project, host, local_settings, credential_dir, runtime = _paired_project( + tmp_path + ) + local_settings.write_bytes(b'{"env": {bad json') + + ok, message = uninstall_claude_hooks( + project_path=project, remove_credentials=True, bridge_config_root=host + ) + + assert not ok + assert "Removed Daem0n hooks from" in message + assert f"Left {local_settings.resolve()} unchanged" in message + assert local_settings.read_bytes() == b'{"env": {bad json' + assert json.loads(fake_settings.read_text())["hooks"] == {} + assert not credential_dir.exists() and not runtime.exists() + + def test_deleted_project_reports_what_was_and_was_not_done( + self, fake_settings, tmp_path + ): + project, host, _local, credential_dir, _runtime = _paired_project(tmp_path) + shutil.rmtree(project) + + ok, message = uninstall_claude_hooks( + project_path=project, remove_credentials=True, bridge_config_root=host + ) + + assert not ok + assert "Removed Daem0n hooks from" in message + assert "Could not locate edit bridge credentials" in message + assert "by hand" in message + assert credential_dir.exists() + + def test_remove_credentials_without_project_says_so(self, fake_settings): + ok, message = uninstall_claude_hooks(remove_credentials=True) + assert ok + assert "No project given; edit bridge credentials kept." in message + + @pytest.mark.parametrize( + "content", + ['{bad json, "theme": "dark"', '{"hooks": []}', '{"hooks": {"Stop": {}}}'], + ) + def test_malformed_user_settings_are_never_overwritten( + self, fake_settings, content + ): + fake_settings.write_text(content, encoding="utf-8") + + installed, install_message = install_claude_hooks() + removed, uninstall_message = uninstall_claude_hooks() + + assert not installed and not removed + assert "Nothing was changed" in install_message + assert "Nothing was changed" in uninstall_message + assert fake_settings.read_text(encoding="utf-8") == content + + def test_non_object_hook_entries_are_kept(self, fake_settings): + fake_settings.write_text( + json.dumps({"hooks": {"Stop": ["odd", {"hooks": "odd"}]}}), + encoding="utf-8", + ) + ok, message = install_claude_hooks() + assert ok, message + stop = json.loads(fake_settings.read_text())["hooks"]["Stop"] + assert stop[:2] == ["odd", {"hooks": "odd"}] diff --git a/tests/test_claude_hooks_post_edit.py b/tests/test_claude_hooks_post_edit.py index 8c56a63..aab0fd6 100644 --- a/tests/test_claude_hooks_post_edit.py +++ b/tests/test_claude_hooks_post_edit.py @@ -1,101 +1,90 @@ """Tests for the post_edit Claude Code hook.""" import json - -import pytest - -from daem0nmcp.claude_hooks.post_edit import main - - -def _set_env(monkeypatch, project_dir, tool_input): - monkeypatch.setenv("CLAUDE_PROJECT_DIR", project_dir) - monkeypatch.setenv("TOOL_INPUT", json.dumps(tool_input)) - - -def test_unapproved_edit_never_emits_a_memory_suggestion(tmp_path, monkeypatch, capsys): - (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "server.py"), - "old_string": "", - "new_string": "class UserAuthService:\n def __init__(self):\n pass\n", - }, +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +PROBE = ( + "import runpy, sys\n" + "try:\n" + " runpy.run_module('daem0nmcp.claude_hooks.post_edit', run_name='__main__')\n" + "except SystemExit as exc:\n" + " code = exc.code\n" + "print(code, 'daem0nmcp.edit_host' in sys.modules)\n" +) + + +def _run(event: dict, extra_env: dict | None = None) -> subprocess.CompletedProcess: + environment = {**os.environ, "PYTHONPATH": str(ROOT)} + environment.pop("DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", None) + environment.update(extra_env or {}) + return subprocess.run( + [sys.executable, "-c", PROBE], + input=json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, ) - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 0 - out = capsys.readouterr().out - assert out == "" +def _event(project: Path) -> dict: + return { + "session_id": "s-1", + "cwd": str(project), + "hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_use_id": "toolu_1", + "tool_input": {"file_path": str(project / "a.py"), "content": "x = 1\n"}, + "tool_response": {"success": True}, + } -def test_trivial_change_no_output(tmp_path, monkeypatch, capsys): +def test_unpaired_project_exits_zero_without_importing_the_bridge(tmp_path): (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "notes.txt"), - "old_string": "hello", - "new_string": "world", - }, - ) - - with pytest.raises(SystemExit) as exc_info: - main() + result = _run(_event(tmp_path)) + assert result.returncode == 0, result.stderr + assert result.stdout.split() == ["0", "False"] - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" - -def test_large_unapproved_change_is_not_captured_from_hook_text( - tmp_path, monkeypatch, capsys -): +def test_misconfigured_credential_still_exits_zero(tmp_path): (tmp_path / ".daem0nmcp").mkdir() - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "big.py"), - "old_string": "", - "new_string": "x = 1\n" * 200, - }, + result = _run( + _event(tmp_path), + {"DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE": str(tmp_path / "missing.json")}, ) + assert result.returncode == 0, result.stderr + assert result.stdout.split() == ["0", "True"] - with pytest.raises(SystemExit) as exc_info: - main() - - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" - - -def test_no_file_path_exits_clean(tmp_path, monkeypatch, capsys): - (tmp_path / ".daem0nmcp").mkdir() - _set_env(monkeypatch, str(tmp_path), {"command": "ls"}) - - with pytest.raises(SystemExit) as exc_info: - main() - - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" +def test_paired_project_without_consumed_edit_captures_nothing(tmp_path, monkeypatch): + from daem0nmcp.claude_hooks.post_edit import handle_post_edit + from daem0nmcp.edit_host import ( + EditHostStateStore, + provision_local_bridge_installation, + ) -def test_no_project_exits_clean(tmp_path, monkeypatch, capsys): - _set_env( - monkeypatch, - str(tmp_path), - { - "file_path": str(tmp_path / "server.py"), - "old_string": "", - "new_string": "class Foo: pass", - }, + project = tmp_path / "project" + (project / ".daem0nmcp").mkdir(parents=True) + installation = provision_local_bridge_installation( + project, config_root=tmp_path / "host" ) + for name, value in installation.environment().items(): + monkeypatch.setenv(name, value) + lookups = [] + original = EditHostStateStore.get_consumed + + def spy(self, **kwargs): + result = original(self, **kwargs) + lookups.append(result) + return result - with pytest.raises(SystemExit) as exc_info: - main() + monkeypatch.setattr(EditHostStateStore, "get_consumed", spy) - assert exc_info.value.code == 0 - assert capsys.readouterr().out == "" + assert handle_post_edit(_event(project), str(project)) is False + assert lookups == [None] diff --git a/tests/test_claude_hooks_pre_edit.py b/tests/test_claude_hooks_pre_edit.py index 7987b59..c6fc1cf 100644 --- a/tests/test_claude_hooks_pre_edit.py +++ b/tests/test_claude_hooks_pre_edit.py @@ -1,109 +1,164 @@ -"""Tests for the pre_edit Claude Code hook (blocking).""" +"""Tests for the pre_edit Claude Code hook (remind only, never block).""" -import pytest -import pytest_asyncio - -from daem0nmcp.claude_hooks.pre_edit import async_main, handle_pre_edit -from daem0nmcp.database import DatabaseManager -from daem0nmcp.edit_bridge_transport import provision_bridge_credential - - -@pytest_asyncio.fixture -async def tmp_project(tmp_path): - """Create a temp project with initialised database.""" - daem0n_dir = tmp_path / ".daem0nmcp" - daem0n_dir.mkdir() - storage = daem0n_dir / "storage" - storage.mkdir() - - db = DatabaseManager(str(storage)) - await db.init_db() - yield tmp_path - await db.close() - - -@pytest.mark.asyncio -async def test_blocks_without_preflight(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" - assert str(tmp_project) not in result.message +import json +import os +import subprocess +import sys +from pathlib import Path +import pytest -@pytest.mark.asyncio -async def test_legacy_context_state_cannot_bypass_v7_preflight(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +from daem0nmcp.claude_hooks._client import relative_project_path + +ROOT = Path(__file__).resolve().parents[1] + + +def _run( + args: list[str], event: dict | str, home: Path, env: dict | None = None +) -> subprocess.CompletedProcess[str]: + # HOME bounds the project search, so an enclosing checkout's .daem0nmcp/ + # (or the developer's CLAUDE_PROJECT_DIR) can never leak into a test. + environment = { + **os.environ, + "PYTHONPATH": str(ROOT), + "HOME": str(home), + "USERPROFILE": str(home), + } + environment.pop("CLAUDE_PROJECT_DIR", None) + environment.update(env or {}) + return subprocess.run( + [sys.executable, *args], + input=event if isinstance(event, str) else json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, + ) -@pytest.mark.asyncio -async def test_permissive_mode_allows_through(tmp_project, monkeypatch): - """In permissive mode the block message is returned but allowed=False still.""" - monkeypatch.setenv("DAEM0N_HOOKS_PERMISSIVE", "1") +def _pre_edit(event: dict | str, home: Path, env: dict | None = None): + return _run(["-m", "daem0nmcp.claude_hooks.pre_edit"], event, home, env) - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - # async_main still returns allowed=False (it doesn't know about permissive mode) - # but the message is what block() would print. main() handles the permissive exit. - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +def _write_event(cwd: Path, file_path: str) -> dict: + return { + "session_id": "s-1", + "transcript_path": str(cwd / "t.jsonl"), + "cwd": str(cwd), + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"file_path": file_path, "content": "x = 1\n"}, + "tool_use_id": "toolu_1", + } -@pytest.mark.asyncio -async def test_does_not_read_or_surface_legacy_file_memories(tmp_project): - file_path = str(tmp_project / "server.py") - result = await async_main(str(tmp_project), file_path) - assert not result.allowed - assert "race condition" not in result.message - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" +def _context(result: subprocess.CompletedProcess[str]) -> str: + return json.loads(result.stdout)["hookSpecificOutput"]["additionalContext"] -def test_no_file_path_exits_clean(tmp_path, monkeypatch): - (tmp_path / ".daem0nmcp").mkdir() - monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path)) - monkeypatch.setenv("TOOL_INPUT", '{"command": "ls -la"}') +@pytest.fixture +def project(tmp_path: Path) -> Path: + root = tmp_path / "project" + (root / ".daem0nmcp").mkdir(parents=True) + return root - from daem0nmcp.claude_hooks.pre_edit import main - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 2 +def test_non_daem0n_directory_exits_zero_silently(tmp_path): + plain = tmp_path / "plain" + plain.mkdir() + result = _pre_edit(_write_event(plain, str(plain / "app.py")), tmp_path) + assert result.returncode == 0 + assert result.stdout == "" -def test_remote_mode_without_workspace_binding_fails_recoverably(tmp_path, monkeypatch): - target = tmp_path / "target.txt" - target.write_text("before", encoding="utf-8") - credential = tmp_path / "host" / "credential.json" - provision_bridge_credential( - credential, - principal_id="remote-principal", - transports=frozenset({"remote-https"}), +def test_daem0n_project_exits_zero_with_reminder(tmp_path, project): + result = _pre_edit(_write_event(project, str(project / "src" / "app.py")), tmp_path) + assert result.returncode == 0 + output = json.loads(result.stdout)["hookSpecificOutput"] + assert output["hookEventName"] == "PreToolUse" + assert "permissionDecision" not in output + assert "memory_preflight" in output["additionalContext"] + assert ( + 'memory_recall_file(relative_file_path="src/app.py")' + in output["additionalContext"] ) - ca_file = tmp_path / "ca.pem" - ca_file.write_text("unused", encoding="utf-8") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE", str(credential)) - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_MODE", "remote-https") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_REMOTE_URL", "https://server.example") - monkeypatch.setenv("DAEM0NMCP_EDIT_BRIDGE_CA_FILE", str(ca_file)) - monkeypatch.delenv("DAEM0NMCP_EDIT_HOST_WORKSPACE_BINDING_FILE", raising=False) - - result = handle_pre_edit( - { - "session_id": "remote-session", - "tool_use_id": "remote-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "before", - "new_string": "after", - }, - }, - str(tmp_path), + assert "\n" not in output["additionalContext"] + + +def test_subdirectory_cwd_uses_the_enclosing_project(tmp_path, project): + sub = project / "src" + sub.mkdir() + result = _pre_edit(_write_event(sub, "app.py"), tmp_path) + assert result.returncode == 0 + assert 'relative_file_path="src/app.py"' in _context(result) + + +def test_falls_back_to_claude_project_dir(tmp_path, project): + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + event = _write_event(elsewhere, str(project / "notes.md")) + result = _pre_edit(event, tmp_path, env={"CLAUDE_PROJECT_DIR": str(project)}) + assert result.returncode == 0 + assert 'relative_file_path="notes.md"' in _context(result) + + +def test_home_directory_global_state_is_not_a_project(tmp_path): + (tmp_path / ".daem0nmcp" / "edit-bridges").mkdir(parents=True) + result = _pre_edit(_write_event(tmp_path, str(tmp_path / "a.py")), tmp_path) + assert result.returncode == 0 + assert result.stdout == "" + + +def test_path_outside_project_and_malformed_input_still_exit_zero(tmp_path, project): + outside = _pre_edit(_write_event(project, str(tmp_path / "x.py")), tmp_path) + assert outside.returncode == 0 + assert "relative_file_path" not in _context(outside) + + garbage = _pre_edit("not json", tmp_path) + assert garbage.returncode == 0 + assert garbage.stdout == "" + + +def test_file_name_is_json_escaped_in_the_reminder(tmp_path, project): + result = _pre_edit(_write_event(project, 'a"b).py'), tmp_path) + assert result.returncode == 0 + assert 'relative_file_path="a\\"b).py"' in _context(result) + + +@pytest.mark.parametrize( + "raw", + [ + r"\\attacker-host\share\x.py", + "//attacker-host/share/x.py", + "../outside.py", + "C:other.py" if os.name == "nt" else "/etc/passwd", + ], +) +def test_relative_path_is_lexical_and_drops_unc_or_outside(tmp_path, monkeypatch, raw): + def no_filesystem(*_args, **_kwargs): + raise AssertionError("model-supplied paths must not touch the filesystem") + + monkeypatch.setattr(Path, "resolve", no_filesystem) + monkeypatch.setattr(os, "stat", no_filesystem) + monkeypatch.setattr(os.path, "realpath", no_filesystem) + assert relative_project_path(tmp_path / "project", raw) is None + assert relative_project_path(tmp_path / "project", "src/../a.py") == "a.py" + + +def test_imports_no_api_or_retrieval_modules(tmp_path, project): + probe = ( + "import sys, runpy\n" + "try:\n" + " runpy.run_module('daem0nmcp.claude_hooks.pre_edit', run_name='__main__')\n" + "except SystemExit:\n" + " pass\n" + "heavy = sorted(m for m in sys.modules if m.startswith((" + "'daem0nmcp.api', 'daem0nmcp.retrieval', 'daem0nmcp.edit_host'," + " 'daem0nmcp.database', 'sqlalchemy', 'pydantic')))\n" + "print(heavy, file=sys.stderr)\n" + "sys.exit(1 if heavy else 0)\n" ) - - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" + result = _run(["-c", probe], _write_event(project, str(project / "a.py")), tmp_path) + assert result.returncode == 0, result.stderr + assert "additionalContext" in result.stdout diff --git a/tests/test_claude_hooks_stop.py b/tests/test_claude_hooks_stop.py index adce7a2..41627eb 100644 --- a/tests/test_claude_hooks_stop.py +++ b/tests/test_claude_hooks_stop.py @@ -1,5 +1,11 @@ """Tests for the stop Claude Code hook.""" +import json +import os +import subprocess +import sys +from pathlib import Path + import pytest import pytest_asyncio @@ -65,7 +71,8 @@ async def test_suggests_decisions_without_writing_memory(tmp_project): ] state = {"reminder_count": 0, "last_reminder_turn": -1} result = await analyse_and_remember(str(tmp_project), messages, state) - assert "hook did not write memory" in result.message + assert "The hook wrote nothing" in result.message + assert "ask Claude" in result.message assert "mcp__daem0nmcp__memory_preflight" in result.message assert "mcp__daem0nmcp__memory_store" in result.message @@ -108,3 +115,102 @@ async def test_no_messages_returns_empty(tmp_project): # 0-length messages -> anti-loop sees turn 0 vs -1, which passes, # but no content means no completion signal -> empty assert result.message == "" + + +def _run_stop(event: dict, home: Path) -> subprocess.CompletedProcess[str]: + environment = { + **os.environ, + "PYTHONPATH": str(Path(__file__).resolve().parents[1]), + "HOME": str(home), + "USERPROFILE": str(home), + } + environment.pop("CLAUDE_PROJECT_DIR", None) + return subprocess.run( + [sys.executable, "-m", "daem0nmcp.claude_hooks.stop"], + input=json.dumps(event), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, + ) + + +def test_main_reads_stdin_event_and_nested_transcript(tmp_path): + project = tmp_path / "project" + (project / ".daem0nmcp").mkdir(parents=True) + (project / "src").mkdir() + state_dir = tmp_path / ".daem0nmcp" / "hook_state" + state_dir.mkdir(parents=True) + stale = state_dir / "stop_old-session.json" + stale.write_text("{}", encoding="utf-8") + os.utime(stale, (0, 0)) + transcript = tmp_path / "transcript.jsonl" + filler = [ + {"type": "user", "message": {"role": "user", "content": f"step {n}"}} + for n in range(300) + ] + records = [ + *filler, + {"type": "summary", "summary": "Caching work"}, + { + "type": "user", + "sessionId": "s-1", + "message": {"role": "user", "content": "Add caching"}, + }, + { + "type": "assistant", + "sessionId": "s-1", + "message": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": ( + "I will use Redis for caching because it gives shared " + "state. Implementation is complete and all tasks are done." + ), + } + ], + }, + }, + ] + transcript.write_text( + "\n".join(json.dumps(record) for record in records) + "\n", encoding="utf-8" + ) + event = { + "session_id": "s-1", + "transcript_path": str(transcript), + "cwd": str(project / "src"), + "hook_event_name": "Stop", + "stop_hook_active": False, + } + + result = _run_stop(event, tmp_path) + + assert result.returncode == 0, result.stderr + assert '"decision"' not in result.stdout + message = json.loads(result.stdout)["systemMessage"] + assert "mcp__daem0nmcp__memory_store" in message + assert "memory_record_outcome" in message + assert not (project / ".daem0nmcp" / "storage").exists() + state = json.loads((state_dir / "stop_s-1.json").read_text(encoding="utf-8")) + assert state["last_reminder_turn"] == len(records) + assert not stale.exists() + + +def test_main_is_silent_outside_a_daem0n_project(tmp_path): + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + json.dumps( + {"message": {"role": "assistant", "content": "All tasks are complete."}} + ) + + "\n", + encoding="utf-8", + ) + result = _run_stop( + {"session_id": "s-2", "transcript_path": str(transcript), "cwd": str(tmp_path)}, + tmp_path, + ) + assert result.returncode == 0 + assert result.stdout == "" diff --git a/tests/test_edit_bridge_transport.py b/tests/test_edit_bridge_transport.py index 15c0e77..a574d17 100644 --- a/tests/test_edit_bridge_transport.py +++ b/tests/test_edit_bridge_transport.py @@ -27,7 +27,6 @@ from daem0nmcp.claude_hooks.post_edit_preflight import ( handle_edit_preflight_response, ) -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.covenant import InvocationScope from daem0nmcp.database import DatabaseManager from daem0nmcp.edit_bridge import ( @@ -51,6 +50,7 @@ provision_remote_bridge_installation, ) from daem0nmcp.workspace import WorkspaceRegistry +from tests.native_edit_host import drive_native_edit SECRET = "bridge-secret-" + "x" * 48 AUTHKEY = b"local-auth-key-" + b"y" * 32 @@ -510,8 +510,8 @@ async def test_remote_hooks_bind_different_desktop_root_to_server_workspace( } try: with patch.dict(os.environ, environment, clear=False): - denied = handle_pre_edit(first_event, str(client_root)) - assert not denied.allowed + denied = drive_native_edit(first_event, str(client_root)) + assert not denied assert ( workspace.workspace_id != WorkspaceRegistry(default_root=client_root).default.workspace_id @@ -567,8 +567,8 @@ async def test_remote_hooks_bind_different_desktop_root_to_server_workspace( ) retry_event = {**first_event, "tool_use_id": "remote-allowed-2"} with patch.dict(os.environ, environment, clear=False): - allowed = handle_pre_edit(retry_event, str(client_root)) - assert allowed.allowed + allowed = drive_native_edit(retry_event, str(client_root)) + assert allowed target.write_text("after", encoding="utf-8") with patch.dict(os.environ, environment, clear=False): assert handle_post_edit(retry_event, str(client_root)) diff --git a/tests/test_edit_host.py b/tests/test_edit_host.py index 8a5e2a0..3243414 100644 --- a/tests/test_edit_host.py +++ b/tests/test_edit_host.py @@ -11,7 +11,6 @@ import pytest -from daem0nmcp.claude_hooks.pre_edit import handle_pre_edit from daem0nmcp.edit_bridge_transport import ( RemoteBridgeHTTPSClient, local_bridge_address, @@ -29,6 +28,7 @@ ) from daem0nmcp.protected_files import verify_owner_only_file from daem0nmcp.workspace import WorkspaceRegistry +from tests.native_edit_host import drive_native_edit def _configuration(tmp_path): @@ -236,21 +236,20 @@ def unexpected_call(*_args, **_kwargs): with monkeypatch.context() as context: for name, value in installation.environment().items(): context.setenv(name, value) - result = handle_pre_edit( - { - "session_id": "replacement-session", - "tool_use_id": "replacement-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "replacement", - "new_string": "changed", + with pytest.raises(ValueError): + drive_native_edit( + { + "session_id": "replacement-session", + "tool_use_id": "replacement-request", + "tool_name": "Edit", + "tool_input": { + "file_path": str(target), + "old_string": "replacement", + "new_string": "changed", + }, }, - }, - str(project), - ) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" + str(project), + ) assert calls == 0 project.rename(tmp_path / "replacement-checkout") (tmp_path / "original-checkout").rename(project) @@ -317,21 +316,20 @@ def unexpected_call(*_args, **_kwargs): with monkeypatch.context() as context: for name, value in environment.items(): context.setenv(name, value) - result = handle_pre_edit( - { - "session_id": "substitution-session", - "tool_use_id": "substitution-request", - "tool_name": "Edit", - "tool_input": { - "file_path": str(target), - "old_string": "before", - "new_string": "after", + with pytest.raises(ValueError): + drive_native_edit( + { + "session_id": "substitution-session", + "tool_use_id": "substitution-request", + "tool_name": "Edit", + "tool_input": { + "file_path": str(target), + "old_string": "before", + "new_string": "after", + }, }, - }, - str(project), - ) - assert not result.allowed - assert result.message == "EDIT_BRIDGE_UNAVAILABLE" + str(project), + ) assert calls == 0 with pytest.raises(ValueError, match="conflicts"): From 3ee4628341fb124cdc69e12caff8db003f816697 Mon Sep 17 00:00:00 2001 From: Devil <69057727+9thLevelSoftware@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:55:00 -0400 Subject: [PATCH 65/72] PR 4/12: Garbage-collect projection generations (and retryable entity recall) (#66) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Brownfield run 3403b448, PR 4 of 12** — audit-and-fix stack for the v7 core (#61/#62). Base: `v7/bf-03-report-transient-failures-as-ret`. ## Plan (contract) ### PR 4: Garbage-collect projection generations and take git out of preflight - Depends on: PR 3 - Files: daem0nmcp/retrieval/projections.py, daem0nmcp/retrieval/specialized_projection.py, daem0nmcp/retrieval/jobs.py, daem0nmcp/retrieval/lexical_config.py, daem0nmcp/verification_v7.py, daem0nmcp/api/v7/production.py, daem0nmcp/api/v7/resource_repository.py, tests/test_retrieval_projection_growth.py, tests/api_v7/test_production.py - Traces to: FP-6, F-001, F-010, F-026, F-039, F-057, F-060, F-040, A-002, A-009, A-019 - Risk: medium - Description: 1. **Generation GC** (KD-5; R-24). - Move `_GENERATION_TABLES` from `verification_v7.py` into `retrieval/lexical_config.py`, alongside the FTS table naming for lexical and procedure. Verification imports it. - Add `collect_superseded_generations(connection, workspace_id, projection, limit=5)`. It opens its own `BEGIN IMMEDIATE`. It selects manifests with `status IN ('ready','failed')` for that workspace and projection, excluding the active manifest and the most recently deactivated (`MAX(activated_at_us)` among `ready`), oldest first, `LIMIT 5`. For each it runs `DROP TABLE IF EXISTS` on the regex-validated FTS table names, deletes the generation-table rows, and deletes the manifest. - Call it *after* the activation commit in `LexicalProjectionBuilder.rebuild` and in the specialized builder activation. - Catch lock/busy errors, log them at debug, and return; the next activation retries. - Next generation numbers stay `MAX(generation)+1`, so deleted manifests below the maximum are harmless. 2. **Lock contention** (F-060). In `rebuild`, classify lock contention with `jobs._is_lock_contention` as retryable instead of `LEXICAL_UNAVAILABLE`. 3. **Preflight** (F-040). Give `production._guidance_reader` a narrow reader in `resource_repository.py` that returns only the active rules and recent decisions/warnings, with no `_read_git_changes_sync`. `session_brief` keeps the git probe. 4. **Tests.** - 50 stores + forced drains leave ≤ 2 generations per projection in `sqlite_master`, ≤ 2 manifests per projection, and `retrieval_documents` ≤ 2 × live records. - A DB seeded with 120 stale generations shrinks over successive writes (5 per activation per projection) to ≤ 2. - A failed or never-activated generation just below the new one never causes the previously active generation to be dropped. - A GC pass while another connection holds a read transaction does not error the write. - `memory_preflight` spawns no subprocess. - Acceptance: - The growth test passes in under 10 s on 3 OSes, including POSIX in DELETE journal mode (pre-PR-8 DBs). - `verify-v7 --repair-projections` tests stay green. - The preflight profile shows no `git` calls. #### Carry-over from PR 3 (orchestrator, binding) - "Take git out of preflight" is DONE in PR 3 (987bd00: `read_briefing_snapshot(include_git_changes=...)`, guidance passes False) — it was the root cause of preflight timeouts. Drop that part of this PR; keep only the acceptance check that preflight spawns no git process, if not already covered. - Pre-existing race (seen on CI ubuntu 3.10 and locally under load): `memory_recall_entity` raises non-retryable CAPABILITY_DEGRADED when the retrieval projection doesn't yet contain every record the entity points to (`discovery_operations.py:2096`). This PR owns projection freshness: make "projection not yet caught up" a retryable outcome (e.g. DATABASE_IN_USE/RETRIEVAL_UNAVAILABLE with retry_after_ms) and make `test_process_discovery_surface` wait/poll for recall to see the records, like `test_process_client` does. - [REASSIGNED to PR 10 by the orchestrator after PR 4's general review: it shares no code or data path with generation GC and holds nothing up] Recall `bounded_excerpt` repeats short content twice (`retrieval/composer.py:561-566`). Also confirm the "LEXICAL_REBUILD_REQUIRED"/stale marker set on every write (`event_store.py:612`) always clears once the rebuild job runs (GC must not leave it stuck). --- ## What landed # PR 4 summary: Garbage-collect projection generations (and retryable entity recall) (Saved by the orchestrator from the implementer's reply.) Branch `v7/bf-04-garbage-collect-projection-gener`, base 987bd00 (PR 3 tip). Worktree `D:\Daem0n-MCP\.claude\worktrees\agent-aacf6400555e70fbc`. - 556c1b4 `fix: garbage-collect superseded projection generations` - 10bf9d6 `fix: report entity recall on a lagging lexical index as retryable` - d0773dd `test: expect collected lexical generations after reactivation` (final) ## What changed 1. **Generation GC (KD-5, R-24; F-001/F-010/F-026/F-039/F-057)**: `GENERATION_TABLES` moved to `retrieval/lexical_config.py` (verification_v7 imports it). New `retrieval/projections.py::collect_superseded_generations(connection, workspace_id, projection, limit=5)`: no-op inside a caller's transaction, for `graph` (discovery FKs), or invalid workspace id; else own `BEGIN IMMEDIATE` with 250 ms busy timeout (restored after); selects manifests `status IN ('ready','failed')`, keeping the most recently deactivated `ready` (highest `activated_at_us`, ties by generation) — active/building/rebuild_required excluded by the filter; oldest first, ≤5 per pass; per generation: `DROP TABLE IF EXISTS` FTS table (`lexical_fts_table_name`/`procedure_fts_table_name`), delete generation-table rows, delete manifest; busy/locked → debug log, other `sqlite3.Error` → warning; returns 0, never raises. Called after the activation commit in `LexicalProjectionBuilder.rebuild` and `SpecializedProjectionBuilder.rebuild` (procedure, outcome, temporal) only when the builder owns the transaction, never on dry runs or when reusing a current generation. 2. **Lock contention (F-060)**: lexical and specialized rebuilds report SQLite busy/locked (`ProjectionJobRunner._is_lock_contention`) as `DATABASE_IN_USE`; job runner re-queues instead of dead-lettering. `api/v7/operations.py` passes projection DATABASE_IN_USE through (projection_rebuild retryable, not CAPABILITY_DEGRADED). 3. **Preflight git**: done in PR 3; acceptance test added. 4. **Entity recall race**: `_memory_recall_entity` returns retryable DATABASE_IN_USE (250 ms) when entity members are missing and the lexical provider isn't `ready`; stays non-retryable CAPABILITY_DEGRADED when lexical is `ready`. `test_process_discovery_surface` polls recall with a 30 s deadline accepting only ok or retryable. ## Tests - New `tests/test_retrieval_projection_growth.py` (forces DELETE journal mode, `synchronous=OFF`): 50 spaced writes via real `drain_projection_jobs` → lexical/procedure/outcome/temporal ≤2 manifests, lexical/procedure ≤2 FTS tables matching manifests, `retrieval_documents` ≤ 2× live records; 120-generation backlog (GC patched out while seeding) shrinks by exactly 5 per activation to {active−1, active} for lexical and procedure; failed/never-activated generations dropped while previous active survives; GC while another connection holds a read transaction on an old FTS table → rebuild still active, GC returns 0 after 250 ms busy, connection clean, next activation collects; contended lexical job → `queued`, reason DATABASE_IN_USE; GC in caller transaction or for graph → no-op. Reverted-source: 5 tests fail (e.g. 51 lexical manifests left). - `test_production.py::test_memory_preflight_spawns_no_subprocess` (control: session_brief spawns git; preflight spawns nothing; fails if guidance sets include_git_changes=True). - `test_discovery_operations.py::test_memory_recall_entity_missing_members_retry_only_while_stale`. - `test_migrate_v7.py` reactivation expects `[(2,ready),(3,active)]` (gen 1 collected). - Full suite Windows 3.12: 2825 passed, 121 skipped, 0 failed (1671 s). Linux Docker 3.10: growth file 6 passed in 3.2 s (<10 s acceptance); jobs/specialized/runtime/verify_v7 (incl. --repair-projections)/discovery/production/process_discovery: 149 passed, 3 skipped. Windows related modules: 504 passed. ruff/format clean; CI mypy clean. - Entity recall race not reproduced in 12+12 parallel runs; classification rests on the carry-over diagnosis and code reading. ## Deviations 1. Outside file list: `api/v7/operations.py` (DATABASE_IN_USE passthrough), `tests/test_migrate_v7.py` (expectation). 2. Specialized builder also got the F-060 fix. 3. GC catches every `sqlite3.Error`. 4. `resource_repository.py`/`production.py` unchanged (PR 3 did the git change). ## Follow-ups (not fixed) - Graph generations still accumulate (discovery rows reference them). - `rebuild_required` manifests from verify-v7 repair never collected (by design). - Existing DBs shrink 5 generations per activation per projection; VACUUM needed to reclaim disk (PR 12 docs). - Lexical provider reads manifest then FTS table in separate implicit transactions; keeping the previous generation makes it safe for one activation; two activations between reads → "unavailable", not a crash. - `_generation_is_current_sync` in recall_entity still non-retryable CAPABILITY_DEGRADED if a graph rebuild lands mid-request. - Carry-over not addressed (added to the plan after launch): recall `bounded_excerpt` repeats short content (`retrieval/composer.py:561-566`); confirm stale/LEXICAL_REBUILD_REQUIRED marker always clears. ## Round 2 (ff9dcf5) 21 addressed, 3 wontfix (bounded_excerpt trio reassigned to PR 10), 1 wontfix (graph GC — discovery FKs; tracked follow-up). - Entity recall: `_lexical_catchup_pending_sync` — retryable only when the missing record_ids are absent from the ACTIVE lexical generation's `retrieval_documents` AND a `retrieval.projection_rebuild` job is queued/running; dead-lettered rebuild, missing FTS5, config/table mismatch, missing diagnostic, or indexed-but-unmatched → terminal CAPABILITY_DEGRADED. Kept DATABASE_IN_USE (RETRIEVAL_UNAVAILABLE isn't in the wire registry). - Growth test: `include_optional=True`, 20 writes, asserts all four projections rebuilt each drain (5.0 s Windows / 2.07 s Linux); asserts no active manifest keeps `rebuild_required_*`. - One-shot compaction: `retrieval.operations.compact_projections` + CLI `python -m daem0nmcp.cli --project-path compact-projections --workspace-id ws_…` (`--no-vacuum` to skip VACUUM). Takes the exclusive storage lock (server must be stopped); loops GC per projection until 0 then VACUUMs; reports collected and before/after superseded counts. Release inventory regenerated. **PR 12 must document this as the step BEFORE VACUUM.** - Keep-rule by `ORDER BY generation DESC` (no clock); `rebuild_required` generations below active now collected (COALESCE guard); per-victim SAVEPOINT so one undroppable victim doesn't wedge GC; handlers catch `Exception`; `typeof(generation)='integer'` and delete by manifest_id; `jobs.is_lock_contention` public; single `COLLECTABLE_PROJECTIONS`; verify-v7 emits `checks["projection_generations"]`; generation-row deletion asserted for all three tables; WAL blocked-reader test; process-surface poll asserts DATABASE_IN_USE. Results: Windows full suite 2830 passed / 121 skipped (2 errors = unrelated disk-I/O flake in test_graph.py under container load; module passes alone). Linux 3.10 container green; ruff/format, CI mypy, release inventory clean. Mutations: GC disabled → 3 tests fail; GENERATION_TABLES shrunk to lexical → growth test fails. --- ## Review # PR 4 review — merged (round 2: verified by tests, general-2 and plan-alignment; CI run 35490179442 green) Reviewer slots: general, general-2, tests, security, plan-alignment (effort 5). Item statuses: 20 addressed, 4 wontfix. CI: https://github.com/9thlevelsoftware/Daem0n-MCP/actions/runs/35490179442 (green) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- daem0nmcp/api/v7/discovery_operations.py | 66 +- daem0nmcp/api/v7/operations.py | 2 + daem0nmcp/cli.py | 33 +- daem0nmcp/retrieval/jobs.py | 39 +- daem0nmcp/retrieval/lexical_config.py | 9 + daem0nmcp/retrieval/operations.py | 80 ++- daem0nmcp/retrieval/projections.py | 159 +++++ daem0nmcp/retrieval/specialized_projection.py | 15 +- daem0nmcp/verification_v7.py | 46 +- docs/release/v7/inventory.md | 3 +- docs/release/v7/requirements.json | 14 +- tests/api_v7/test_discovery_operations.py | 79 +++ .../api_v7/test_process_discovery_surface.py | 30 +- tests/api_v7/test_production.py | 49 ++ tests/test_cli_retrieval.py | 61 ++ tests/test_migrate_v7.py | 3 +- tests/test_retrieval_projection_growth.py | 570 ++++++++++++++++++ tests/test_verify_v7.py | 27 + 18 files changed, 1242 insertions(+), 43 deletions(-) create mode 100644 tests/test_retrieval_projection_growth.py diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..3a5c52c 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -2046,6 +2046,56 @@ def reader(connection: sqlite3.Connection) -> _EntitySelection: raise _translate_error(error) from None +def _lexical_catchup_pending_sync( + dependencies: DiscoveryOperationDependencies, + workspace: Workspace, + record_ids: tuple[str, ...], +) -> bool: + """Return whether the lexical projection still owes these records. + + True only when the records are absent from the active lexical generation + *and* a rebuild job is queued or running, i.e. a retry can actually make + progress. A dead-lettered rebuild, a permanently unavailable index, or a + record that simply does not match the entity name is not retryable. + """ + + if not record_ids: + return False + + def reader(connection: sqlite3.Connection) -> bool: + active = connection.execute( + "SELECT generation FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND status='active'", + (workspace.workspace_id,), + ).fetchone() + indexed = 0 + if active is not None: + placeholders = ",".join("?" for _ in record_ids) + indexed = int( + connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + f"AND projection_generation=? AND record_id IN ({placeholders})", + (workspace.workspace_id, int(active[0]), *record_ids), + ).fetchone()[0] + ) + if indexed >= len(record_ids): + return False + return ( + connection.execute( + "SELECT 1 FROM background_jobs WHERE workspace_id=? " + "AND job_type='retrieval.projection_rebuild' " + "AND status IN ('queued','running') LIMIT 1", + (workspace.workspace_id,), + ).fetchone() + is not None + ) + + try: + return bool(_read_snapshot(dependencies, workspace, reader)) + except Exception: + return False + + def _generation_is_current_sync( dependencies: DiscoveryOperationDependencies, workspace: Workspace, @@ -2093,7 +2143,21 @@ async def _memory_recall_entity( raise DiscoveryOperationError("CAPABILITY_DEGRADED") indexed[record_id] = item.record if set(indexed) != set(selection.record_ids): - raise DiscoveryOperationError("CAPABILITY_DEGRADED") + missing = tuple( + record_id + for record_id in selection.record_ids + if record_id not in indexed + ) + # Only a projection that has genuinely not caught up yet is worth + # retrying. A permanently unavailable or dead-lettered lexical + # index stays a terminal CAPABILITY_DEGRADED. + pending = await _run_blocking( + dependencies, + lambda: _lexical_catchup_pending_sync(dependencies, workspace, missing), + ) + raise DiscoveryOperationError( + "DATABASE_IN_USE" if pending else "CAPABILITY_DEGRADED" + ) await _run_blocking( dependencies, lambda: _generation_is_current_sync( diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..14420ef 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -912,6 +912,8 @@ def _projection_sync( raise CoreOperationError("CAPABILITY_DISABLED") from exc if exc.code in {"LEXICAL_UNAVAILABLE", "FTS5_UNAVAILABLE"}: raise CoreOperationError("LEXICAL_UNAVAILABLE") from exc + if exc.code == "DATABASE_IN_USE": + raise CoreOperationError("DATABASE_IN_USE") from exc raise CoreOperationError("CAPABILITY_DEGRADED") from exc except CoreOperationError: raise diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index efa4eb6..ed157ed 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -378,6 +378,19 @@ def main(): "--workspace-id", required=True, help="Registered opaque workspace ID" ) + compact_projections_parser = subparsers.add_parser( + "compact-projections", + help="Offline: drop every superseded projection generation and VACUUM", + ) + compact_projections_parser.add_argument( + "--workspace-id", required=True, help="Registered opaque workspace ID" + ) + compact_projections_parser.add_argument( + "--no-vacuum", + action="store_true", + help="Drop the superseded generations without reclaiming the file", + ) + rebuild_projection_parser = subparsers.add_parser( "rebuild-projection", help="Build and activate one v7 retrieval projection" ) @@ -770,11 +783,16 @@ def main(): print(f"Runs: {len(payload['runs'])}") sys.exit(exit_code) - if args.command in {"projection-status", "rebuild-projection"}: + if args.command in { + "compact-projections", + "projection-status", + "rebuild-projection", + }: import sqlite3 from .retrieval.operations import ( ProjectionOperationError, + compact_projections, projection_status, rebuild_projection, ) @@ -802,7 +820,12 @@ def main(): update={"project_root": str(workspace.root)} ) storage_path = Path(workspace_settings.get_storage_path()) - with DatabaseFileLock(storage_path, "shared"): + # Compaction drops generations a running server could be reading, + # so it takes the offline exclusive lock. + lock_mode = ( + "exclusive" if args.command == "compact-projections" else "shared" + ) + with DatabaseFileLock(storage_path, lock_mode): active = resolve_active_database(storage_path) if active.format_version != 7: raise ProjectionOperationError("FORMAT_7_REQUIRED") @@ -813,6 +836,12 @@ def main(): connection.execute("PRAGMA foreign_keys=ON") if args.command == "projection-status": payload = projection_status(connection, workspace.workspace_id) + elif args.command == "compact-projections": + payload = compact_projections( + connection, + workspace.workspace_id, + vacuum=not args.no_vacuum, + ) else: builders = create_projection_builders( connection, diff --git a/daem0nmcp/retrieval/jobs.py b/daem0nmcp/retrieval/jobs.py index 4376bbd..f9f7a60 100644 --- a/daem0nmcp/retrieval/jobs.py +++ b/daem0nmcp/retrieval/jobs.py @@ -26,6 +26,26 @@ _HEARTBEAT_SHUTDOWN_GRACE_SECONDS = 1.5 +def is_lock_contention(exc: BaseException) -> bool: + """Return whether *exc* is SQLite lock contention a caller may retry.""" + + if not isinstance(exc, sqlite3.OperationalError): + return False + code = getattr(exc, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + } + # sqlite_errorcode is unavailable on supported Python 3.10 builds. + # Match only SQLite's fixed lock diagnostics, never arbitrary detail. + return str(exc).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + class ProjectionJobError(RuntimeError): """Owned, sanitized projection-job failure.""" @@ -250,7 +270,7 @@ def _heartbeat_loop( break except sqlite3.OperationalError as exc: connection.rollback() - if not self._is_lock_contention(exc): + if not is_lock_contention(exc): raise if stop.is_set(): if stop_seen_at is None: @@ -554,22 +574,6 @@ def _now(self) -> int: raise ProjectionJobError("PROJECTION_JOB_CLOCK_INVALID") return value - @staticmethod - def _is_lock_contention(exc: sqlite3.OperationalError) -> bool: - code = getattr(exc, "sqlite_errorcode", None) - if isinstance(code, int): - return code & 0xFF in { - sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_LOCKED, - } - # sqlite_errorcode is unavailable on supported Python 3.10 builds. - # Match only SQLite's fixed lock diagnostics, never arbitrary detail. - return str(exc).casefold() in { - "database is locked", - "database table is locked", - "database schema is locked", - } - def create_projection_job_runner( connection: sqlite3.Connection, @@ -593,4 +597,5 @@ def create_projection_job_runner( "ProjectionJobRun", "ProjectionJobRunner", "create_projection_job_runner", + "is_lock_contention", ] diff --git a/daem0nmcp/retrieval/lexical_config.py b/daem0nmcp/retrieval/lexical_config.py index d23c243..e87797f 100644 --- a/daem0nmcp/retrieval/lexical_config.py +++ b/daem0nmcp/retrieval/lexical_config.py @@ -18,6 +18,14 @@ "tokenizer": LEXICAL_TOKENIZER, } +# Per-generation row tables of the rebuildable local projections. Temporal +# keeps no rows of its own; graph generations are owned by discovery tables. +GENERATION_TABLES = { + "lexical": "retrieval_documents", + "procedure": "record_procedures", + "outcome": "record_outcome_view", +} + _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _FTS_PARTITION = re.compile(r"^retrieval_fts_[0-9a-f]{24}_g[1-9][0-9]*$") @@ -48,6 +56,7 @@ def lexical_fts_table_name(workspace_id: str, generation: int) -> str: __all__ = [ + "GENERATION_TABLES", "LEXICAL_BM25_WEIGHTS", "LEXICAL_BUILD_CONFIG", "LEXICAL_TOKENIZER", diff --git a/daem0nmcp/retrieval/operations.py b/daem0nmcp/retrieval/operations.py index ab48297..08935e2 100644 --- a/daem0nmcp/retrieval/operations.py +++ b/daem0nmcp/retrieval/operations.py @@ -9,7 +9,12 @@ from dataclasses import asdict from typing import Any -from .projections import LexicalProjectionBuilder, ProjectionBuildError +from .projections import ( + COLLECTABLE_PROJECTIONS, + LexicalProjectionBuilder, + ProjectionBuildError, + collect_superseded_generations, +) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _PROJECTIONS = frozenset( @@ -157,8 +162,81 @@ def projection_status( } +def compact_projections( + connection: sqlite3.Connection, + workspace_id: str, + *, + vacuum: bool = True, +) -> dict[str, Any]: + """Drain every superseded generation offline, then reclaim the file. + + Online activations collect at most five generations each, so a database + that grew before the garbage collector existed needs about a hundred + writes to drain — and a ``VACUUM`` before that only rewrites the dead + generations. This runs the same collector in a loop until it reports + nothing left, so the following ``VACUUM`` actually reclaims the space. + The caller owns the offline exclusive lock; no activation may run + concurrently. + """ + + if not isinstance(connection, sqlite3.Connection): + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + _validate_workspace(workspace_id) + if connection.in_transaction: + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + collected: dict[str, int] = {} + try: + before = _generation_counts(connection, workspace_id) + for projection in sorted(COLLECTABLE_PROJECTIONS): + removed = 0 + while True: + passed = collect_superseded_generations( + connection, workspace_id, projection + ) + if passed == 0: + break + removed += passed + collected[projection] = removed + if vacuum: + connection.execute("VACUUM") + after = _generation_counts(connection, workspace_id) + except sqlite3.Error as exc: + raise ProjectionOperationError("PROJECTION_COMPACTION_FAILED") from exc + return { + "collected": collected, + "superseded_manifests_after": after[0], + "superseded_manifests_before": before[0], + "vacuumed": bool(vacuum), + "workspace_id": workspace_id, + } + + +def _generation_counts( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[int, int]: + placeholders = ",".join("?" for _ in COLLECTABLE_PROJECTIONS) + projections = sorted(COLLECTABLE_PROJECTIONS) + superseded = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders}) AND status<>'active'", + (workspace_id, *projections), + ).fetchone()[0] + ) + total = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders})", + (workspace_id, *projections), + ).fetchone()[0] + ) + return superseded, total + + __all__ = [ "ProjectionOperationError", + "compact_projections", "projection_status", "rebuild_projection", ] diff --git a/daem0nmcp/retrieval/projections.py b/daem0nmcp/retrieval/projections.py index 4114d29..2548719 100644 --- a/daem0nmcp/retrieval/projections.py +++ b/daem0nmcp/retrieval/projections.py @@ -4,6 +4,7 @@ import hashlib import json +import logging import re import sqlite3 import time @@ -11,14 +12,22 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention from .lexical_config import ( + GENERATION_TABLES, LEXICAL_TOKENIZER, lexical_build_config_hash, lexical_fts_table_name, ) +from .specialized_contract import procedure_fts_table_name + +logger = logging.getLogger(__name__) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _LEXICAL_BUILDER_VERSION = "retrieval-lexical-1" +# Graph is excluded: the discovery tables reference its generations. +COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) +_GC_BUSY_TIMEOUT_MS = 250 class ProjectionBuildError(RuntimeError): @@ -284,11 +293,17 @@ def rebuild( self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") if isinstance(exc, ProjectionBuildError): raise + if is_lock_contention(exc): + raise ProjectionBuildError( + "DATABASE_IN_USE", "lexical projection build was contended" + ) from exc if isinstance(exc, sqlite3.Error): raise ProjectionBuildError( "LEXICAL_UNAVAILABLE", "FTS5 projection build is unavailable" ) from exc raise + if owns_transaction: + collect_superseded_generations(self.connection, workspace_id, "lexical") return result def apply_active_delta(self, workspace_id: str, record_id: str) -> bool: @@ -748,8 +763,152 @@ def _clock_value(self) -> int: return value +_VICTIM_QUERY = """ +SELECT manifest_id, generation FROM projection_manifests +WHERE workspace_id=? AND projection_name=? AND typeof(generation)='integer' + AND ( + ( + status IN ('ready','failed') + AND manifest_id IS NOT ( + SELECT manifest_id FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='ready' + AND activated_at_us IS NOT NULL + ORDER BY generation DESC LIMIT 1 + ) + ) + OR ( + status='rebuild_required' + AND generation < COALESCE(( + SELECT generation FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='active' + AND typeof(generation)='integer' + ), -1) + ) + ) +ORDER BY generation LIMIT ? +""" + + +def _drop_generation( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + manifest_id: str, + generation: int, +) -> None: + fts_table = ( + lexical_fts_table_name(workspace_id, generation) + if projection == "lexical" + else procedure_fts_table_name(workspace_id, generation) + if projection == "procedure" + else None + ) + if fts_table is not None: + connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') + generation_table = GENERATION_TABLES.get(projection) + if generation_table is not None: + connection.execute( + f'DELETE FROM "{generation_table}" ' + "WHERE workspace_id=? AND projection_generation=?", + (workspace_id, generation), + ) + connection.execute( + "DELETE FROM projection_manifests WHERE manifest_id=?", + (manifest_id,), + ) + + +def collect_superseded_generations( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + limit: int = 5, +) -> int: + """Drop up to *limit* superseded generations of one local projection. + + Runs after an activation commits, in its own short transaction. Victims + are chosen by manifest status: ``ready`` and ``failed`` rows, plus + ``rebuild_required`` rows below the active generation (readers consult + those only while no active manifest exists). It never touches the active + or a ``building`` generation, nor the most recently deactivated one (the + highest activated ``ready`` generation), because a reader that resolved + the old manifest just before the switch may still be querying it. + + Best effort: one damaged generation is skipped rather than wedging the + rest, contention ends the pass, and nothing raises — the next activation + retries. Returns the number of generations removed. + """ + + if ( + projection not in COLLECTABLE_PROJECTIONS + or _WORKSPACE_ID.fullmatch(workspace_id) is None + or connection.in_transaction + ): + return 0 + collected = 0 + try: + previous_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0]) + connection.execute(f"PRAGMA busy_timeout={_GC_BUSY_TIMEOUT_MS}") + try: + connection.execute("BEGIN IMMEDIATE") + try: + victims = connection.execute( + _VICTIM_QUERY, + ( + workspace_id, + projection, + workspace_id, + projection, + workspace_id, + projection, + limit, + ), + ).fetchall() + for manifest_id, generation in victims: + connection.execute("SAVEPOINT retrieval_generation_gc") + try: + _drop_generation( + connection, + workspace_id, + projection, + str(manifest_id), + int(generation), + ) + except Exception as exc: + connection.execute( + "ROLLBACK TO SAVEPOINT retrieval_generation_gc" + ) + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + if is_lock_contention(exc): + raise + # A damaged generation must not wedge every later + # one; skip past it and keep collecting. + logger.warning( + "projection generation could not be collected", + exc_info=True, + ) + continue + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + collected += 1 + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.execute(f"PRAGMA busy_timeout={previous_timeout}") + except Exception as exc: + if is_lock_contention(exc): + logger.debug("projection generation GC skipped: database busy") + else: + logger.warning("projection generation GC failed", exc_info=True) + return 0 + return collected + + __all__ = [ + "COLLECTABLE_PROJECTIONS", "LexicalProjectionBuilder", "ProjectionBuildError", "ProjectionBuildResult", + "collect_superseded_generations", ] diff --git a/daem0nmcp/retrieval/specialized_projection.py b/daem0nmcp/retrieval/specialized_projection.py index a5d8043..1282ee5 100644 --- a/daem0nmcp/retrieval/specialized_projection.py +++ b/daem0nmcp/retrieval/specialized_projection.py @@ -11,6 +11,8 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention +from .projections import collect_superseded_generations from .specialized_contract import ( SPECIALIZED_BUILDER_VERSION, specialized_projection_contract, @@ -357,17 +359,28 @@ def rebuild( self.connection.commit() else: self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") - return result except Exception as exc: self._rollback(owns_transaction, savepoint) if isinstance(exc, SpecializedProjectionBuildError): raise + if is_lock_contention(exc): + raise SpecializedProjectionBuildError( + "DATABASE_IN_USE", + "specialized projection build was contended", + ) from exc if isinstance(exc, sqlite3.Error): raise SpecializedProjectionBuildError( "PROJECTION_UNAVAILABLE", "specialized projection build is unavailable", ) from exc raise + if owns_transaction: + # Graph is not collectable: the discovery tables reference its + # generations. collect_superseded_generations enforces that. + collect_superseded_generations( + self.connection, workspace_id, projection_name + ) + return result def active_is_current(self, workspace_id: str, projection_name: str) -> bool: """Return whether the active specialized generation is exact.""" diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index be1003b..eed1d3d 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -35,6 +35,7 @@ ) from .migrations import MIGRATIONS from .migrations.v7 import _source_row_hash, inventory_database +from .retrieval.lexical_config import GENERATION_TABLES from .schema_version import CURRENT_SCHEMA_VERSION, REQUIRED_V7_SCHEMA_VERSIONS from .storage_activation import ( ActiveDatabasePointer, @@ -92,11 +93,6 @@ "entities", "communities", } -_GENERATION_TABLES = { - "lexical": "retrieval_documents", - "procedure": "record_procedures", - "outcome": "record_outcome_view", -} class VerificationV7Error(RuntimeError): @@ -664,6 +660,34 @@ def _verify_mappings(connection: sqlite3.Connection, replay: sqlite3.Connection) return count +def _generation_inventory(connection: sqlite3.Connection) -> dict[str, int]: + """Count superseded local generations so a stalled GC is visible.""" + + from .retrieval.projections import COLLECTABLE_PROJECTIONS + + projections = sorted(COLLECTABLE_PROJECTIONS) + placeholders = ",".join("?" for _ in projections) + superseded = { + str(row[0]): int(row[1]) + for row in connection.execute( + "SELECT projection_name,COUNT(*) FROM projection_manifests " + f"WHERE projection_name IN ({placeholders}) AND status<>'active' " + "GROUP BY projection_name", + projections, + ) + } + inventory = {name: superseded.get(name, 0) for name in projections} + inventory["fts_partitions"] = int( + connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*' " + "AND (name GLOB 'retrieval_fts_*_g*' " + "OR name GLOB 'retrieval_procedure_fts_*_g*')" + ).fetchone()[0] + ) + return inventory + + def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: from .retrieval.projections import LexicalProjectionBuilder from .retrieval.specialized_projection import SpecializedProjectionBuilder @@ -679,7 +703,7 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: "source_event_root_hash,details_json,generation,row_count," "cursor_recorded_at_us,cursor_event_id FROM projection_manifests" ).fetchall() - for name, generation_table in _GENERATION_TABLES.items(): + for name, generation_table in GENERATION_TABLES.items(): if connection.execute( f'SELECT 1 FROM "{generation_table}" rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " @@ -934,6 +958,14 @@ def _verify_database( ) except Exception as exc: checks["projection_manifests"] = _check(False, error=type(exc).__name__) + try: + # Superseded generations should stay near zero. A number that + # keeps growing means the activation-time GC is failing. + checks["projection_generations"] = _check( + True, **_generation_inventory(source) + ) + except Exception as exc: + checks["projection_generations"] = _check(False, error=type(exc).__name__) try: invalid = int( source.execute( @@ -1115,7 +1147,7 @@ def _refresh_manifests(candidate: Path, workspace_ids: list[str]) -> None: ) # Remove only orphaned local generations in the candidate. Otherwise a # renamed/deleted manifest can leave an FTS table occupying the next ID. - for name, table in _GENERATION_TABLES.items(): + for name, table in GENERATION_TABLES.items(): connection.execute( f'DELETE FROM "{table}" AS rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " diff --git a/docs/release/v7/inventory.md b/docs/release/v7/inventory.md index db82039..cc683e0 100644 --- a/docs/release/v7/inventory.md +++ b/docs/release/v7/inventory.md @@ -2,7 +2,7 @@ Generated by `python scripts/v7_release_inventory.py --write`. Registration is not acceptance; only an `accepted` requirement with explicit evidence metadata can be accepted. -Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **571**. +Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **572**. ## Release ledger @@ -131,6 +131,7 @@ Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resou | hook | `hooks/daem0n_stop_hook.py` | `hooks/daem0n_stop_hook.py` | Hook hooks/daem0n_stop_hook.py runs with its documented input and redacted failure behavior. — pending evidence | pending | | cli | `briefing` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli briefing parses and exercises its documented result contract. — pending evidence | pending | | cli | `check` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli check parses and exercises its documented result contract. — pending evidence | pending | +| cli | `compact-projections` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract. — pending evidence | pending | | cli | `index` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli index parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-claude-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-claude-hooks parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-hooks parses and exercises its documented result contract. — pending evidence | pending | diff --git a/docs/release/v7/requirements.json b/docs/release/v7/requirements.json index 6346d97..8b452d4 100644 --- a/docs/release/v7/requirements.json +++ b/docs/release/v7/requirements.json @@ -4370,6 +4370,16 @@ "package": "daem0nmcp.cli", "status": "pending" }, + { + "acceptance": "python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract.", + "entry_point": "daem0nmcp.cli:main", + "evidence": [], + "id": "cli.compact-projections", + "kind": "cli", + "owner": "cli", + "package": "daem0nmcp.cli", + "status": "pending" + }, { "acceptance": "python -m daem0nmcp.cli index parses and exercises its documented result contract.", "entry_point": "daem0nmcp.cli:main", @@ -5744,8 +5754,8 @@ "tests/test_legacy_vector_migration_guards.py" ], "observed_surface": { - "atomic_requirements": 571, - "cli_commands": 20, + "atomic_requirements": 572, + "cli_commands": 21, "dependency_profiles": 10, "hooks": 15, "registered_tools": 75, diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index a40b2d8..eab8404 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1162,6 +1162,85 @@ async def retrieve(self, workspace, query, linked_workspace_ids): self.assertEqual(1, len(second.items)) self.assertNotEqual(first.items[0].record_id, second.items[0].record_id) + async def test_memory_recall_entity_retries_only_a_pending_catch_up( + self, + ) -> None: + """Only a projection that still owes the records is worth retrying.""" + from daem0nmcp.api.v7.discovery_operations import DiscoveryOperationError + from daem0nmcp.api.v7.models import ( + ProviderDiagnostic, + RetrievalData, + TokenUsage, + ) + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._activate_discovery() + + class RecallService: + """Returns nothing, as a lexical miss on the entity name does.""" + + async def retrieve(self, workspace, query, linked_workspace_ids): + return RetrievalData( + provider_diagnostics=[ + ProviderDiagnostic( + provider="lexical", + status="unavailable", + manifest_generation=None, + elapsed_ms=0.0, + reason="LEXICAL_UNAVAILABLE", + returned_count=0, + ) + ], + abstained=True, + abstention_reason="NO_EVIDENCE", + token_usage=TokenUsage( + budget=query.token_budget, + requested=0, + selected=0, + rendered=0, + dropped=0, + ), + ) + + operation = self._operations(recall_service=RecallService())[ + "memory_recall_entity" + ] + request = _request( + "memory_recall_entity", + workspace_id=self.workspace.workspace_id, + entity_name="Authentication", + ) + + async def code() -> str: + with self.assertRaises(DiscoveryOperationError) as raised: + await operation(workspace=self.workspace, request=request) + return raised.exception.code + + # The records are not in the lexical projection yet and its rebuild + # job is queued: a retry can make progress. + self.assertEqual("DATABASE_IN_USE", await code()) + + # The rebuild dead-lettered, so nothing will catch up on its own. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute( + "UPDATE background_jobs SET status='dead_letter'," + "finished_at_us=1 WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + + # The projection holds every member, so the miss is genuine even + # though lexical reports itself unavailable and a job is queued. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + LexicalProjectionBuilder(connection).rebuild(self.workspace.workspace_id) + connection.execute( + "UPDATE background_jobs SET status='queued',finished_at_us=NULL " + "WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + async def test_stats_read_the_active_canonical_graph_snapshot(self) -> None: """Counting retained-v6 graph rows would misreport the v7 projection.""" from daem0nmcp.api.v7.tools import KnowledgeGraphStatsData diff --git a/tests/api_v7/test_process_discovery_surface.py b/tests/api_v7/test_process_discovery_surface.py index 2b3f101..60e9377 100644 --- a/tests/api_v7/test_process_discovery_surface.py +++ b/tests/api_v7/test_process_discovery_surface.py @@ -2,13 +2,14 @@ from __future__ import annotations +import asyncio import importlib.util import pytest from daem0nmcp.database import DatabaseManager from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed @pytest.fixture @@ -132,15 +133,24 @@ async def test_production_graph_discovery_surface(initialized_workspace, transpo {**scope, "entity_id": entity["entity_id"]}, ) assert traced["entity"]["entity_id"] == entity["entity_id"] - recalled = await succeed( - session, - "memory_recall_entity", - {**scope, "entity_name": entity["name"]}, - ) - assert {item["record_id"] for item in recalled["items"]} & { - first["record_id"], - second["record_id"], - } + # The lexical projection may still be catching up with the link and + # backfill events; recall reports that as retryable until it has. + recall_deadline = asyncio.get_running_loop().time() + 30 + while True: + recalled = await call( + session, + "memory_recall_entity", + {**scope, "entity_name": entity["name"]}, + ) + if recalled["ok"] and { + item["record_id"] for item in recalled["data"]["items"] + } & {first["record_id"], second["record_id"]}: + break + if not recalled["ok"]: + assert recalled["error"]["code"] == "DATABASE_IN_USE", recalled + assert recalled["error"]["retryable"], recalled + assert asyncio.get_running_loop().time() < recall_deadline, recalled + await asyncio.sleep(0.25) graph = await succeed( session, "knowledge_graph_get", diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index 1c71d37..7bfc48b 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -602,5 +602,54 @@ def build_server(_surface, **options): self.assertEqual([False], observed) +async def test_memory_preflight_spawns_no_subprocess(tmp_path): + """Preflight guidance must not pay for git (F-040); session_brief still does.""" + import subprocess + + from fastmcp import Client + + from daem0nmcp.api.v7.production import create_v7_server + from tests.api_v7.process_client import initialize_workspaces + + [workspace] = await initialize_workspaces((tmp_path,)) + spawned: list[list[str]] = [] + + def record_spawn(argv, *args, **kwargs): + spawned.append(list(argv)) + raise FileNotFoundError(argv[0]) + + server = create_v7_server( + "stdio", + settings=Settings( + project_root=str(tmp_path), + workspace_roots=[str(tmp_path)], + dream_enabled=False, + ), + environ={}, + ) + scope = {"workspace_id": workspace.workspace_id} + async with Client(server) as client: + with patch.object(subprocess, "Popen", side_effect=record_spawn): + brief = await client.call_tool("session_brief", scope, raise_on_error=False) + assert brief.structured_content["ok"], brief.structured_content + assert any(argv[0] == "git" for argv in spawned), spawned + spawned.clear() + preflight = await client.call_tool( + "memory_preflight", + { + **scope, + "target_tool": "memory_store", + "target_arguments": { + "record_type": "decision", + "content": "Preflight must not spawn processes.", + "idempotency_key": "preflight-no-subprocess-0001", + }, + }, + raise_on_error=False, + ) + assert preflight.structured_content["ok"], preflight.structured_content + assert spawned == [] + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cli_retrieval.py b/tests/test_cli_retrieval.py index 057f712..5c60a42 100644 --- a/tests/test_cli_retrieval.py +++ b/tests/test_cli_retrieval.py @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from tests.test_migrate_v7 import _create_legacy_database @@ -157,6 +158,66 @@ def test_registered_secondary_workspace_opens_its_own_active_database(self): self.assertEqual(1, len(lexical)) self.assertEqual(second_id, payload["workspace_id"]) + def test_compact_projections_drains_superseded_generations_offline(self): + """Databases bloated before the GC existed need a one-shot drain.""" + + import sqlite3 + + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.storage_activation import resolve_active_database + + with tempfile.TemporaryDirectory() as raw: + root, workspace_id = self._migrated_fixture(raw) + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(9): + LexicalProjectionBuilder(connection).rebuild(workspace_id) + superseded = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical' AND status<>'active'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(9, superseded) + + result = self._run( + root, + "compact-projections", + "--workspace-id", + workspace_id, + ) + + self.assertEqual(0, result.returncode, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(8, payload["collected"]["lexical"]) + self.assertLess( + payload["superseded_manifests_after"], + payload["superseded_manifests_before"], + ) + self.assertTrue(payload["vacuumed"]) + self.assertNotIn(str(root), result.stdout) + connection = sqlite3.connect(database) + try: + remaining = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical'" + ).fetchone()[0] + tables = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND name GLOB 'retrieval_fts_*_g*' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(2, remaining) + self.assertEqual(2, tables) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index cb4969d..5710696 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -1320,8 +1320,9 @@ def test_reactivation_preserves_only_the_previous_active_lexical_generation(self self.assertEqual("reactivate", reactivated.action) active = sqlite3.connect(resolve_active_database(storage).path) try: + # Generation 1 is superseded twice over and is collected. self.assertEqual( - [(1, "ready"), (2, "ready"), (3, "active")], + [(2, "ready"), (3, "active")], active.execute( "SELECT generation,status FROM projection_manifests " "WHERE projection_name='lexical' ORDER BY generation" diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py new file mode 100644 index 0000000..64014fb --- /dev/null +++ b/tests/test_retrieval_projection_growth.py @@ -0,0 +1,570 @@ +"""Superseded projection generations are garbage-collected (KD-5, F-001). + +Every write rebuilds the local projections into a new generation. Without GC +each rebuild left a full-corpus FTS5 table, its document rows and a manifest +behind, so storage grew quadratically with the number of memories. +""" + +from __future__ import annotations + +import re +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +WORKSPACE_ID = "ws_0123456789abcdef01234567" +_NO_OPTIONAL_PROFILES = { + "local": "disabled", + "models-local": "disabled", + "graph": "disabled", +} +_LOCAL = ("lexical", "procedure", "outcome", "temporal") +# Spelled out here on purpose: importing the production mapping would make a +# shrunken GENERATION_TABLES pass this file. +_GENERATION_TABLES = { + "lexical": "retrieval_documents", + "outcome": "record_outcome_view", + "procedure": "record_procedures", +} + + +def _record(content: str, *, procedure: bool = False) -> dict[str, object]: + return { + "record_type": "procedure" if procedure else "decision", + "legacy_type": None, + "content": content, + "rationale": None, + "context": {"steps": ["Check state.", "Apply change."]} if procedure else {}, + "tags": ["growth"], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "growth-test", + "source_model": None, + "deleted_at_us": None, + } + + +class ProjectionGenerationGrowthTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + from daem0nmcp.migrations.schema import MIGRATIONS + + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "growth.sqlite3" + self.connection = sqlite3.connect(self.path) + self.connection.execute("PRAGMA foreign_keys=ON") + # Pre-PR-8 databases on POSIX still use the rollback journal, where + # readers block a committing writer; GC must stay safe there. + self.connection.execute("PRAGMA journal_mode=DELETE") + # Durability is irrelevant here; skipping fsync keeps the hundreds of + # seeding rebuilds fast on slow CI disks. + self.connection.execute("PRAGMA synchronous=OFF") + for version, _description, statements in MIGRATIONS: + if 16 <= version <= 18: + for statement in statements: + self.connection.execute(statement) + self.connection.commit() + self._sequence = 0 + + def tearDown(self) -> None: + self.connection.close() + self._directory.cleanup() + + def _record_outcome(self, record_id: str) -> None: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.outcome_recorded", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": { + **_record("growth memory number 1", procedure=True), + "outcome": "the generation was collected", + "worked": True, + } + }, + ) + ) + self.connection.commit() + + def _append(self, *, procedure: bool = False) -> str: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + record_id = f"mem_{self._sequence:064x}" + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": _record( + f"growth memory number {self._sequence}", procedure=procedure + ) + }, + ) + ) + self.connection.commit() + return record_id + + def _rebuild(self, projection: str) -> None: + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.retrieval.specialized_projection import ( + SpecializedProjectionBuilder, + ) + + if projection == "lexical": + LexicalProjectionBuilder(self.connection).rebuild(WORKSPACE_ID) + else: + SpecializedProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID, projection, force=True + ) + + def _manifests(self, projection: str) -> dict[int, str]: + return { + int(row[0]): str(row[1]) + for row in self.connection.execute( + "SELECT generation,status FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=?", + (WORKSPACE_ID, projection), + ) + } + + def _fts_generations(self, projection: str) -> set[int]: + prefix = ( + "retrieval_procedure_fts_" + if projection == "procedure" + else ("retrieval_fts_") + ) + pattern = re.compile(re.escape(prefix + WORKSPACE_ID[3:]) + r"_g(\d+)") + return { + int(match[1]) + for (name,) in self.connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + if (match := pattern.fullmatch(str(name))) is not None + } + + def _active(self, projection: str) -> int: + [generation] = [ + generation + for generation, status in self._manifests(projection).items() + if status == "active" + ] + return generation + + async def test_spaced_writes_keep_generations_bounded(self) -> None: + from daem0nmcp.retrieval.runtime import drain_projection_jobs + + first = self._append(procedure=True) + self._record_outcome(first) + for projection in _LOCAL: + self._rebuild(projection) + writes = 20 + for index in range(writes): + self._append(procedure=index % 5 == 0) + # Every job is due immediately: this is the interactive case of + # writes spaced further apart than the coalescing grace. + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runs = await drain_projection_jobs( + self.path, + config=SimpleNamespace(), + max_jobs=8, + # The real drain: every local projection rebuilds, so the + # specialized GC call site runs on the production path. + include_optional=True, + capability_statuses=_NO_OPTIONAL_PROFILES, + ) + self.assertEqual( + {"succeeded"}, + {run.status for run in runs}, + [run.reason for run in runs], + ) + self.assertEqual( + set(_LOCAL), + {name for run in runs for name in run.projections}, + ) + + for projection in _LOCAL: + manifests = self._manifests(projection) + self.assertLessEqual(len(manifests), 2, (projection, manifests)) + self.assertEqual(1, list(manifests.values()).count("active"), projection) + for projection in ("lexical", "procedure"): + tables = self._fts_generations(projection) + self.assertLessEqual(len(tables), 2, (projection, tables)) + self.assertEqual(tables, set(self._manifests(projection)), projection) + live = self.connection.execute( + "SELECT COUNT(*) FROM memory_records WHERE workspace_id=?", + (WORKSPACE_ID,), + ).fetchone()[0] + self.assertEqual(writes + 1, live) + documents = self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents" + ).fetchone()[0] + self.assertLessEqual(documents, 2 * live) + self._assert_generation_rows_collected() + # The stale marker every write sets clears once the rebuild runs, and + # GC never leaves it stuck on an active manifest. + self.assertEqual( + [], + self.connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active' AND (" + "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " + "OR json_extract(details_json,'$.rebuild_required_at_us') IS NOT NULL)", + (WORKSPACE_ID,), + ).fetchall(), + ) + + def _assert_generation_rows_collected(self) -> None: + """Every generation table keeps rows only for surviving manifests.""" + + for projection, table in _GENERATION_TABLES.items(): + rows = { + int(row[0]) + for row in self.connection.execute( + f'SELECT DISTINCT projection_generation FROM "{table}" ' + "WHERE workspace_id=?", + (WORKSPACE_ID,), + ) + } + self.assertTrue(rows, f"{table} has no rows to prove collection") + self.assertLessEqual(rows, set(self._manifests(projection)), table) + + def test_stale_backlog_shrinks_five_generations_per_activation(self) -> None: + self._append(procedure=True) + for projection in ("lexical", "procedure"): + with self.subTest(projection=projection): + # Seed 120 superseded generations the way a pre-GC database + # accumulated them. + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(121): + self._rebuild(projection) + self.assertEqual(121, len(self._manifests(projection))) + self.assertEqual(121, len(self._fts_generations(projection))) + + previous = 120 + while previous > 1: + self._rebuild(projection) + stale = sum( + status != "active" + for status in self._manifests(projection).values() + ) + self.assertEqual(max(1, previous + 1 - 5), stale) + previous = stale + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests)) + self.assertEqual(set(manifests), self._fts_generations(projection)) + active = self._active(projection) + self.assertEqual({active - 1, active}, set(manifests)) + + def test_failed_or_unactivated_generation_never_evicts_last_active(self) -> None: + from daem0nmcp.event_store import deterministic_id + + self._append() + self._rebuild("lexical") + self._rebuild("lexical") + previous_active = self._active("lexical") + template = self.connection.execute( + "SELECT source_event_count,source_event_root_hash,row_count," + "builder_version FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, previous_active), + ).fetchone() + # Generations just below the next one that never became active: an + # arithmetic "keep the one below the new generation" rule would keep + # these and evict the generation readers were actually using. + for offset, status in ((1, "failed"), (2, "ready")): + generation = previous_active + offset + self.connection.execute( + "INSERT INTO projection_manifests (manifest_id,workspace_id," + "projection_name,generation,projection_version,status," + "source_event_count,source_event_root_hash,row_count," + "builder_version,details_json,started_at_us) " + "VALUES (?,?,'lexical',?,1,?,?,?,?,?,'{}',1)", + ( + deterministic_id("prj", "growth-test", generation), + WORKSPACE_ID, + generation, + status, + *template, + ), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + active = self._active("lexical") + self.assertEqual(previous_active + 3, active) + self.assertEqual({previous_active: "ready", active: "active"}, manifests) + self.assertEqual({previous_active, active}, self._fts_generations("lexical")) + + def test_gc_blocked_by_an_open_reader_does_not_fail_the_rebuild(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + reader = sqlite3.connect(self.path, check_same_thread=False) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + # A reader that looked up an old generation is mid-query when + # the activation's GC starts. + reader.execute("BEGIN") + reader.execute( + f'SELECT COUNT(*) FROM "{lexical_fts_table_name(WORKSPACE_ID, oldest)}"' + ).fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + self.assertEqual([0], collected) + self.assertIn(oldest, self._manifests("lexical")) + self.assertIn(oldest, self._fts_generations("lexical")) + self.assertFalse(self.connection.in_transaction) + finally: + reader.close() + + # The next activation retries and catches up. + self._rebuild("lexical") + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertLessEqual(len(self._manifests("lexical")), 2) + + def test_gc_in_wal_mode_collects_under_an_open_reader(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self.connection.execute("PRAGMA journal_mode=WAL") + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + table = lexical_fts_table_name(WORKSPACE_ID, oldest) + reader = sqlite3.connect(self.path) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + reader.execute("BEGIN") + reader.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + # WAL readers do not block the drop, and the reader's snapshot + # still answers from the generation being collected. + self.assertEqual([1], collected) + self.assertEqual( + 1, len(reader.execute(f'SELECT * FROM "{table}" LIMIT 1').fetchall()) + ) + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertNotIn(oldest, self._fts_generations("lexical")) + finally: + reader.close() + + def test_repair_left_rebuild_required_generations_are_collected(self) -> None: + self._append() + self._rebuild("lexical") + repaired = self._active("lexical") + # verify-v7 --repair-projections demotes the active generation and + # rebuilds in one transaction, so its GC is a no-op. + self.connection.execute( + "UPDATE projection_manifests SET status='rebuild_required' " + "WHERE workspace_id=? AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, repaired), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + self.assertNotIn(repaired, manifests) + self.assertEqual(set(manifests), self._fts_generations("lexical")) + self.assertEqual( + 0, + self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + "AND projection_generation=?", + (WORKSPACE_ID, repaired), + ).fetchone()[0], + ) + + def test_an_undroppable_generation_does_not_wedge_the_rest(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + with patch.object( + projections, "collect_superseded_generations", return_value=0 + ): + for _ in range(4): + self._rebuild("lexical") + generations = sorted(self._manifests("lexical")) + damaged = generations[0] + original = projections._drop_generation + + def refuse_oldest( + connection, workspace_id, projection, manifest_id, generation + ): + if generation == damaged: + raise sqlite3.OperationalError("malformed database schema") + return original( + connection, workspace_id, projection, manifest_id, generation + ) + + with patch.object(projections, "_drop_generation", refuse_oldest): + collected = projections.collect_superseded_generations( + self.connection, WORKSPACE_ID, "lexical" + ) + + manifests = self._manifests("lexical") + # The damaged generation is skipped, not retried forever, and every + # later victim is still collected. + self.assertGreaterEqual(collected, 1) + self.assertIn(damaged, manifests) + self.assertIn(damaged, self._fts_generations("lexical")) + self.assertNotIn(generations[1], manifests) + self.assertNotIn(generations[1], self._fts_generations("lexical")) + self.assertIsNotNone(lexical_fts_table_name(WORKSPACE_ID, damaged)) + + def test_offline_compaction_drains_a_backlog_and_reclaims_the_file(self) -> None: + from daem0nmcp.retrieval.operations import compact_projections + + self._append(procedure=True) + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(30): + for projection in ("lexical", "procedure"): + self._rebuild(projection) + self.assertEqual(30, len(self._manifests("lexical"))) + before = self.path.stat().st_size + + payload = compact_projections(self.connection, WORKSPACE_ID) + + self.assertEqual(28, payload["collected"]["lexical"]) + self.assertEqual(28, payload["collected"]["procedure"]) + self.assertTrue(payload["vacuumed"]) + self.assertEqual(58, payload["superseded_manifests_before"]) + self.assertEqual(2, payload["superseded_manifests_after"]) + for projection in ("lexical", "procedure"): + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests), projection) + self.assertEqual(set(manifests), self._fts_generations(projection)) + self.assertLess(self.path.stat().st_size, before) + + def test_lock_contention_is_retried_not_dead_lettered(self) -> None: + from daem0nmcp.retrieval.jobs import ProjectionJobRunner + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._append() + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runner_connection = sqlite3.connect(self.path, timeout=0.1) + blocker = sqlite3.connect(self.path, check_same_thread=False) + lexical = LexicalProjectionBuilder(runner_connection) + + def contended_rebuild(workspace_id: str): + blocker.execute("BEGIN IMMEDIATE") + try: + return lexical.rebuild(workspace_id) + finally: + blocker.rollback() + + try: + run = ProjectionJobRunner( + runner_connection, builders={"lexical": contended_rebuild} + ).run_once() + assert run is not None + self.assertEqual("DATABASE_IN_USE", run.reason) + self.assertEqual("queued", run.status) + finally: + blocker.close() + runner_connection.close() + + +class GenerationGcUnitTests(unittest.TestCase): + def test_gc_inside_a_caller_transaction_is_a_no_op(self) -> None: + from daem0nmcp.retrieval.projections import collect_superseded_generations + + connection = sqlite3.connect(":memory:") + try: + connection.execute("CREATE TABLE t(x)") + connection.execute("INSERT INTO t VALUES (1)") + self.assertTrue(connection.in_transaction) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "lexical") + ) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "graph") + ) + finally: + connection.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_v7.py b/tests/test_verify_v7.py index c50ed20..3ad9705 100644 --- a/tests/test_verify_v7.py +++ b/tests/test_verify_v7.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch import pytest @@ -198,6 +199,32 @@ def test_verify_v7_replays_every_authority_domain(tmp_path: Path) -> None: assert report["checks"]["active_pointer"]["generation"] == 1 +def test_verify_v7_reports_superseded_generation_counts(tmp_path: Path) -> None: + """A GC that has stopped making progress must be visible without logs.""" + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + storage, database = _storage(tmp_path) + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(3): + LexicalProjectionBuilder(connection).rebuild(WORKSPACE_ID) + connection.commit() + + report = verify_v7(storage, WORKSPACE_ID) + + inventory = report["checks"]["projection_generations"] + assert inventory["ok"] is True + # Three generations: one active, two superseded and uncollected. + assert inventory["lexical"] == 2 + assert inventory["outcome"] == 0 + # Only the partitions themselves, never their FTS5 shadow tables. + assert inventory["fts_partitions"] == 3 + + def test_verify_v7_accepts_intact_active_code_generation(tmp_path: Path) -> None: storage, database = _storage(tmp_path) _build_code_projection(database) From 789ceac69b871a4ef8e8af512281c46bc65784a0 Mon Sep 17 00:00:00 2001 From: Devil <69057727+9thLevelSoftware@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:55:01 -0400 Subject: [PATCH 66/72] PR 9/12: Auto-enable installed profiles, fix the Windows launcher and OpenCode installer (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Brownfield run 3403b448, PR 9 of 12** — audit-and-fix stack for the v7 core (#61/#62). Base: `v7/bf-01-make-ci-green-and-pin-workspace`. ## Plan (contract) ### PR 9: Auto-enable installed profiles and fix the Windows launcher and OpenCode installer - Depends on: PR 1 - Files: daem0nmcp/capabilities.py, daem0nmcp/api/v7/discovery_operations.py, daem0nmcp/cli.py, start_server.py, start_daem0nmcp_server.bat, daem0nmcp/opencode_install.py, tests/test_capabilities.py, tests/test_start_server.py, tests/test_opencode_install.py - Traces to: FP-6, FP-7, F-036, F-083, F-032, F-049, F-077, A-016, A-017, UD-7 - Risk: medium - Description (R-9: runtime behavior split out of the docs PR): 1. **Profiles** (KD-8, UD-7). `capabilities._configured` returns "auto" when the variable is unset: - auto → ready if all distributions are present, disabled otherwise; - explicit false → disabled; - explicit true with packages missing → degraded. The disabled-but-installed remediation says to unset the variable or set it to true. Fix the `index` remediation (`tree-sitter-language-pack`) and `discovery_operations.py:1427`. 2. **Launcher** (F-032/F-049). - `start_server.py` uses `--project`, else an existing `DAEM0NMCP_PROJECT_ROOT`, else cwd, and never overwrites an exported value. - `start_daem0nmcp_server.bat` captures `%CD%` before `cd /d "%~dp0"` (or takes `%1`) and passes `--project`. 3. **OpenCode** (F-077). `opencode_install.py` copies the four repo command files into `.opencode/commands/`, create-if-absent. 4. **Windows stdio** (A-016). - Confirm that the stdio variant of `tests/api_v7/test_process_client.py` runs, not skipped, on the Windows leg. - Run the manual Claude Code check on Windows: `claude mcp add daem0nmcp -- python -m daem0nmcp` → `session_brief` → store → recall. Record the Claude Code version and the result in the PR. PR 12 documents it. 5. **Tests.** - `test_capabilities.py` covers all four profile states. - A launcher test: an exported `DAEM0NMCP_PROJECT_ROOT` survives, and `--project` wins. - The installer creates the command files and does not overwrite existing ones. - Acceptance: - A green 3-OS CI run with `[graph]` (installed by CI) now auto-enabled across the whole suite. Any test that assumed "disabled" is fixed here. - `start_daem0nmcp_server.bat` launched from a project directory registers that project. - The PR records the A-016 result. --- ## What landed # PR 9 summary: Auto-enable installed profiles and fix the Windows launcher and OpenCode installer (Saved by the orchestrator from the implementer's reply.) Branch `v7/bf-09-auto-enable-installed-profiles-a`, base f7f024a (PR 1 tip). Worktree `D:\Daem0n-MCP\.claude\worktrees\agent-a51f5956496f2f3bf`. - 6eb091e `feat: enable an optional profile automatically when its extra is installed` - e63f1e0 `fix: make the Windows launcher serve the caller's project` - f2e6a5d `fix: install the OpenCode slash-command files` - 5e731b1 `test: disable installed profiles explicitly where a test needs them off` - 46008ec `test: import the launcher before snapshotting sys.modules` (final) ## What and why 1. **Profiles (KD-8, UD-7)**, `capabilities.py`: `_configured` returns "auto" when the variable is unset or empty (empty used to mean false). unset + installed → ready; unset + missing → disabled with `install_extra`; `false` → disabled with `enable_configuration` ("Unset X or set it to true…"), packages not probed; `true` + missing → degraded (unchanged); invalid → failed. models-local on <3.11 keeps `upgrade_python` (disabled under auto, degraded when true). All consumers go through `CapabilityRegistry`. 2. **Remediation text**: discovery_operations apps remediation ("pip install 'daem0nmcp[apps]'; turns on automatically unless DAEM0NMCP_APPS_ENABLED=false"); `cli.py index` names `tree-sitter-language-pack` and `daem0nmcp[apps]`. 3. **Launcher (F-032, F-049)**: `start_server.py` uses `--project`, else a non-empty exported `DAEM0NMCP_PROJECT_ROOT`, else cwd; never overwrites an exported value. `.bat` captures `%CD%` or `%~f1` before `cd /d "%~dp0"`, passes `--project "%DAEM0N_PROJECT%\."`; header no longer says HTTP is required; CRLF kept. Live check: real `.bat` from a temp project → `session_brief` on the project ok; on the repo's workspace → UNAUTHORIZED_WORKSPACE. 4. **OpenCode (F-077)**: the four command files packaged under `daem0nmcp/opencode_assets/commands/` (package-data `commands/*.md`); `install_opencode` writes them via `_ensure_file` (existing kept `[exists]`, `--force` overwrites, dry-run writes nothing); byte-equality test against repo `.opencode/commands`. 5. **Windows stdio (A-016) — confirmed working**: `test_process_client.py` stdio variant runs (6 passed, 0 skipped) incl. brief → preflight → store → outcome → recall → restart → recall; scripted MCP SDK stdio_client with `python -m daem0nmcp` in a fresh project (initialize 13.2 s cold, 75 tools, brief/preflight/store/recall ok; graph and apps auto-ready); real Claude Code 2.1.278 `claude mcp add` → Connected, `claude -p --mcp-config` brief/preflight/store/recall ok (entry removed afterwards). Line for PR 12: "Stdio (`python -m daem0nmcp`) works on Windows 11 with Claude Code 2.1.278; HTTP via the launcher is optional." ## Tests - `test_capabilities.py`: 5 new (auto+installed ready; auto+missing disabled/install; false+installed disabled/unset message; true+missing degraded; auto models-local on 3.10 disabled/upgrade_python). - New `test_start_server.py`: exported root survives; `--project` wins; cwd default; Windows-only: real `.bat` via `cmd /c` with python.cmd shim checks `%CD%` and a `%1` with spaces/parentheses reach `--project`. - `test_opencode_install.py`: command files created, existing `recall.md` kept, others match repo. - Tests that assumed "disabled" now set the variable false: `test_graph_operations.py`, `test_process_code_impact.py` (also new remediation text), `test_process_dreaming.py`, `test_vectors.py`. - `test_process_portability.py[*-streamable-http]`: with graph auto-on, `storage.rename()` right after HTTP server exit hit WinError 5 intermittently (1/3); passes 4/4 with graph/apps off. Test-only rename retry (15 s deadline) → 5/5. Root cause not pinned. - Isolation sweep: no EXPECTED pin changed. - Full suite (Windows 3.12, `.[dev,apps,graph]`) at 5e731b1: 2813 passed, 121 skipped, 0 failed (1263 s). First run before test fixes: 8 failed (6 "disabled" assumptions + portability rename + one-off consolidation INTERNAL_ERROR, 8/8 on rerun). ruff/format clean; mypy api/v7 clean; release inventory valid. Linux Docker python:3.10-slim affected modules: 115 passed, 5 skipped, 2 failed in test_start_server (stale module after `patch.dict(sys.modules)`) → fixed in 46008ec (3 passed, 2 skipped). ## Deviations 1. Command files packaged (pip installs have no repo) with an equality test. 2. Empty `DAEM0NMCP_

_ENABLED=` treated as auto. 3. Manual check via `claude mcp add`/`list` + headless `claude -p`, not interactive. 4. Edited tests outside file list (graph_operations, process_code_impact, process_dreaming, vectors, process_portability) and pyproject package-data. ## Follow-ups (not fixed) - Portability rename race on Windows with graph on (lingering handle — langgraph checkpoint / Leiden / antivirus?). - `production._capability_states` reports generic "Review the capability profile." remediation in system_health. - `.opencode/plugins/daem0n.ts` also duplicated in the package (kept in sync by test). - Recall on a fresh workspace reported LEXICAL_REBUILD_REQUIRED/stale projections, and `bounded_excerpt` repeated content twice (not investigated). - Stdio cold start ~13 s on Windows. - `.gitignore` lacks `.venv/`. ## Rounds 2–3 2c6ac4d/a68b205/7df3bc2 addressed all 26 review items (22 addressed, 3 partially, 1 wontfix), then 0543b14 merged PR 3 and 2dc3d23 merged PR 4 — PR 9 now stacks on v7/bf-04. - **Root cause of the recurring export→import IMPORT_INVALID (hit CI on PRs 1, 3, 5, 9): SQLITE_BUSY, not a bad bundle.** `prepare_import` does session bookkeeping in short autocommit write transactions after `claim_import_finalization` commits; a concurrent writer (projection drain / dreaming job, busier with graph auto-on) holds the write lock past the 5 s busy_timeout (`portable_projections.py:410 _account_physical_files`). `operations._portable_failure_code()` now walks `__cause__` (only), maps busy/locked via PR 3's `errors.is_database_busy` to retryable DATABASE_IN_USE, and logs the cause on both import and export; the test retries only that code, bounded to 8 attempts/30 s and asserting success within the first 4. - Per-profile preload at build time: a broken-but-installed extra (import raises, e.g. a DLL OSError) degrades only that profile everywhere and names the module and exception type; numpy removed from the graph preload (false warning on every default start). - `capability_states` render the registry's real remediation; `sandbox_execute_python` requires the agency-e2b capability so `=false` really disables it; launcher refuses a non-existent `--project`, uses `setlocal DisableDelayedExpansion`, omits `--project` when an exported root exists, refuses System32; `_ensure_file` refuses symlinks and creates exclusively; invalid-value message mentions "leave it unset". - CHANGELOG "Behaviour change" entry (auto-on profiles, startup/download/re-embed costs, empty = auto, per-profile opt-out). Process harness dumps the server log tail on failure. - Results: Windows full suite 2854 passed / 122 skipped; discovery surface 9/9 across 3 parallel lanes after the PR 4 merge; CI run 35501238206 green on all 9 test legs + lint/type-check/release-inventory. --- ## Review # PR 9 review — merged (round 3: all addressed/wontfix; PR 3 and PR 4 merged in; CI run 35501238206 green on all 12 jobs) Reviewer slots: general, general-2, tests, security, plan-alignment (effort 5). Item statuses: 25 addressed, 1 wontfix. CI: https://github.com/9thlevelsoftware/Daem0n-MCP/actions/runs/35501238206 (green) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 21 +++ daem0nmcp/api/v7/discovery_operations.py | 5 +- daem0nmcp/api/v7/external_operations.py | 24 +++ daem0nmcp/api/v7/operations.py | 48 +++++- daem0nmcp/api/v7/production.py | 98 +++++++---- daem0nmcp/capabilities.py | 44 ++++- daem0nmcp/cli.py | 9 +- daem0nmcp/opencode_assets/commands/commune.md | 13 ++ daem0nmcp/opencode_assets/commands/counsel.md | 24 +++ .../opencode_assets/commands/inscribe.md | 31 ++++ daem0nmcp/opencode_assets/commands/recall.md | 14 ++ daem0nmcp/opencode_install.py | 62 +++++-- pyproject.toml | 2 +- start_daem0nmcp_server.bat | 37 +++- start_server.py | 16 +- tests/api_v7/process_client.py | 158 +++++++++++------- tests/api_v7/test_discovery_operations.py | 24 +++ tests/api_v7/test_entrypoints.py | 9 +- tests/api_v7/test_external_operations.py | 40 +++++ tests/api_v7/test_graph_operations.py | 7 +- tests/api_v7/test_portable_projections.py | 21 +++ tests/api_v7/test_process_client.py | 19 +++ tests/api_v7/test_process_code_impact.py | 8 +- tests/api_v7/test_process_dreaming.py | 2 + tests/api_v7/test_process_portability.py | 87 +++++++--- tests/api_v7/test_production.py | 42 ++++- tests/test_capabilities.py | 90 ++++++++++ tests/test_opencode_install.py | 78 +++++++++ tests/test_start_server.py | 142 ++++++++++++++++ tests/test_vectors.py | 18 +- 30 files changed, 1026 insertions(+), 167 deletions(-) create mode 100644 daem0nmcp/opencode_assets/commands/commune.md create mode 100644 daem0nmcp/opencode_assets/commands/counsel.md create mode 100644 daem0nmcp/opencode_assets/commands/inscribe.md create mode 100644 daem0nmcp/opencode_assets/commands/recall.md create mode 100644 tests/test_start_server.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c67c0..2a1f23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ ### Added - Reproducible v7 core foundation with opt-in dependency profiles and lazy capability reporting. +### Changed +- **Behaviour change — installed profiles now turn themselves on.** An unset (or + empty) `DAEM0NMCP__ENABLED` means "auto": the profile is ready when + its extra is installed and disabled otherwise. `false` still turns it off, and + `true` with packages missing is still degraded. An empty value used to mean + `false`. What this costs if you installed an extra and never set the flag: + `[graph]` imports networkx, igraph and leidenalg at startup, adds a graph + provider to recall (so rankings change) and lets dreaming refresh communities; + `[local]`/`[models-local]` import sentence-transformers and onnxruntime at + startup, download the embedding model on first use, and rebuild the dense + projection, which re-embeds the whole workspace once. Opt out per profile with + `DAEM0NMCP__ENABLED=false`. +- `sandbox_execute_python` now honours `DAEM0NMCP_AGENCY_E2B_ENABLED=false`; it + previously required only `E2B_API_KEY`. +- `system_health` and `meta.capability_states` report the profile's real + remediation (install command or environment variable) instead of "Review the + <name> capability profile." +- `start_daem0nmcp_server.bat` serves the directory you run it from (or its first + argument, or an exported `DAEM0NMCP_PROJECT_ROOT`) instead of the Daem0n-MCP + checkout, and `install-opencode` installs the four `.opencode/commands/` files. + ## [6.0.0] - 2026-01-29 ### Breaking Changes diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 3a5c52c..28f2f29 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -1424,7 +1424,10 @@ def _code_capability_error( else "CAPABILITY_DEGRADED" ) remediation = ( - "Set DAEM0NMCP_APPS_ENABLED=true and install the apps dependency profile." + "Set DAEM0NMCP_APPS_ENABLED to true, false, or leave it unset." + if capability_status == "failed" + else "Install the apps profile with pip install 'daem0nmcp[apps]'; " + "it turns on automatically unless DAEM0NMCP_APPS_ENABLED=false." ) return DiscoveryOperationError( code, diff --git a/daem0nmcp/api/v7/external_operations.py b/daem0nmcp/api/v7/external_operations.py index 57d8001..f07abcb 100644 --- a/daem0nmcp/api/v7/external_operations.py +++ b/daem0nmcp/api/v7/external_operations.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import Any, Protocol +from ...capabilities import CapabilityRegistry from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET @@ -528,6 +529,29 @@ async def sandbox_execute_python( ), ), ) + capability = CapabilityRegistry(environ=dependencies.environment).get( + "agency-e2b" + ) + if capability["status"] != "ready": + # The advertised opt-out must also switch off remote code execution. + raise ExternalOperationError( + "CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + capability_states=( + CapabilityState( + name="agency-e2b", + status=capability["status"], + reason_code="CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + remediation=( + "Install 'daem0nmcp[agency-e2b]' and leave " + "DAEM0NMCP_AGENCY_E2B_ENABLED unset or true." + ), + ), + ), + ) provider = ( dependencies.sandbox_provider_factory() if dependencies.sandbox_provider_factory diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index 14420ef..b155135 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -11,6 +11,7 @@ import asyncio import hashlib import hmac +import logging import os import re import secrets @@ -50,7 +51,7 @@ ) from ...workspace import Workspace from .application import AdmittedRequest -from .errors import STABLE_ERROR_CODE_SET +from .errors import STABLE_ERROR_CODE_SET, is_database_busy from .models import EvidenceRef, Page, RecordSummary from .portable_projections import ( ImportFinalizationLease, @@ -138,6 +139,8 @@ # This is intentionally not the event loop's default executor. The semaphore # remains owned by the concurrent future after an asyncio waiter is cancelled, # so cancellation cannot release capacity while SQLite still holds a lock. +_LOGGER = logging.getLogger(__name__) + _CORE_OPERATION_WORKERS = BoundedWorkerPool( max_workers=4, thread_name_prefix="daem0nmcp-v7-core", @@ -218,6 +221,31 @@ def _validated_storage_path( return resolved +def _portable_failure_code(error: BaseException, operation: str) -> str: + """Name a transfer failure the caller can act on, and log its cause. + + A concurrent writer (a projection drain or a dreaming job) can hold the + SQLite write lock past the busy timeout while a transfer does its session + bookkeeping. That is a transient, retryable condition, not an invalid + bundle, and reporting it as one leaves the caller with no next step. + Only the explicit ``raise ... from`` chain counts: a bundle rejected while + an unrelated busy error is in flight is still an invalid bundle. + """ + + seen: set[int] = set() + cause: BaseException | None = error + while cause is not None and id(cause) not in seen: + seen.add(id(cause)) + if is_database_busy(cause): + _LOGGER.warning("%s deferred: %s", operation, cause) + return "DATABASE_IN_USE" + cause = cause.__cause__ + _LOGGER.warning( + "%s rejected the bundle: %s", operation, type(error).__name__, exc_info=error + ) + return "IMPORT_INVALID" + + def _verify_schema(connection: sqlite3.Connection) -> None: try: row = connection.execute( @@ -1030,9 +1058,14 @@ def _export_sync( except CoreOperationError: raise except PortableTransferError as exc: - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_export") + raise CoreOperationError(code) from exc except Exception as exc: - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_export") + ) from exc def _journal_payload(bundle: ExportBundle, merge: bool) -> dict[str, Any]: @@ -1448,7 +1481,10 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_import") + raise CoreOperationError(code) from exc except CoreOperationError: if connection.in_transaction: connection.rollback() @@ -1476,7 +1512,9 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_import") + ) from exc finally: if lease is not None and not committed: with suppress(Exception): diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index c114041..c12fc56 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -10,7 +10,7 @@ import os import re import secrets -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from contextlib import asynccontextmanager, suppress from dataclasses import dataclass from datetime import datetime, timezone @@ -225,34 +225,46 @@ async def aclose(self) -> None: await await_projection_job_drains(tuple(self._paths)) +# Only entry points the profile's own extra installs; a module from another +# profile would fail here and degrade a working one. +_NATIVE_PROFILE_MODULES = { + "local": ("qdrant_client",), + "models-local": ("sentence_transformers", "onnx", "onnxruntime"), + "graph": ("networkx", "igraph", "leidenalg"), +} + + class _OptionalNativeRuntimeLifecycle: """Initialize enabled native provider entry points before worker threads.""" - def __init__(self, capability_statuses: Mapping[str, str]) -> None: - self._local_enabled = capability_statuses.get("local") == "ready" - self._models_enabled = capability_statuses.get("models-local") == "ready" - self._graph_enabled = capability_statuses.get("graph") == "ready" + def __init__(self, capability_statuses: MutableMapping[str, str]) -> None: + self._statuses = capability_statuses + self.failures: dict[str, str] = {} def start(self) -> None: - modules = [] - if self._local_enabled: - modules.append("qdrant_client") - if self._models_enabled: - modules.extend(("sentence_transformers", "onnx", "onnxruntime")) - if self._graph_enabled: - modules.extend(("numpy", "networkx", "igraph", "leidenalg")) - for module in modules: - # Import the public model/runtime entry points on the main thread. - # This initializes NumPy/SciPy/native DLL dependencies before a - # dense query can race another provider's first worker creation. - try: - importlib.import_module(module) - except Exception: - # Independent providers still get initialized if one is broken. - # Their operation boundary reports stable degraded diagnostics. - logging.getLogger(__name__).warning( - "Optional native provider runtime preload is unavailable" - ) + for profile, modules in _NATIVE_PROFILE_MODULES.items(): + if self._statuses.get(profile) != "ready": + continue + for module in modules: + # Import the public model/runtime entry points on the main + # thread. This initializes NumPy/SciPy/native DLL dependencies + # before a dense query can race another provider's first worker. + try: + importlib.import_module(module) + except Exception as error: + # Installed metadata said ready, but the package cannot run + # here. Report that once instead of per tool call. + self._statuses[profile] = "degraded" + self.failures[profile] = ( + f"importing {module} raised {type(error).__name__}" + ) + logging.getLogger(__name__).warning( + "Optional %s runtime is unavailable: importing %s raised %s", + profile, + module, + type(error).__name__, + ) + break def _loopback_host(host: str) -> bool: @@ -304,29 +316,56 @@ def _task_configuration( ) +def _remediation_text(name: str, remediation: Mapping[str, Any]) -> str: + """Render the registry's structured remediation as one actionable line.""" + + parts = [ + str(remediation[key]) + for key in ("message", "command", "environment") + if remediation.get(key) + ] + missing = remediation.get("missing") + if isinstance(missing, list) and missing: + parts.append("Missing: " + ", ".join(str(item) for item in missing[:8]) + ".") + return " ".join(parts) or f"Review the {name} capability profile." + + def _capability_states( environ: Mapping[str, str], + native_failures: Mapping[str, str] = MappingProxyType({}), ) -> tuple[CapabilityState, ...]: values: list[CapabilityState] = [] for name, capability in CapabilityRegistry(environ=environ).all().items(): status = str(capability["status"]) - if status == "ready": + failure = native_failures.get(name) + if status == "ready" and failure is None: values.append( CapabilityState.model_validate({"name": name, "status": "ready"}) ) continue + if failure is not None: + status = "degraded" reason = { "disabled": "CAPABILITY_DISABLED", "degraded": "CAPABILITY_DEGRADED", "failed": "CAPABILITY_CONFIGURATION_INVALID", }[status] + remediation = capability.get("remediation") + text = ( + f"The {name} profile is installed but cannot run here: {failure}. " + "Reinstall or repair it." + if failure is not None + else _remediation_text( + name, remediation if isinstance(remediation, Mapping) else {} + ) + ) values.append( CapabilityState.model_validate( { "name": name, "status": status, "reason_code": reason, - "remediation": f"Review the {name} capability profile.", + "remediation": text, } ) ) @@ -710,6 +749,10 @@ def _assemble( name: str(capability["status"]) for name, capability in CapabilityRegistry(environ=env).all().items() } + # Preload before anything snapshots the statuses, so an installed but + # broken extra degrades everywhere instead of only failing per call. + native_runtimes = _OptionalNativeRuntimeLifecycle(capability_statuses) + native_runtimes.start() normalizer = build_argument_normalizer() registry = WorkspaceRegistry.from_settings(loaded_settings) workspaces = {registry.default.workspace_id: registry.default} @@ -838,7 +881,7 @@ def schedule_capture_projection(path: Path) -> None: else "jwt" ), task_support=task_state, - capability_states=_capability_states(env), + capability_states=_capability_states(env, native_runtimes.failures), storage_resolver=storage_resolver, dreaming_provider=dreaming.health, runtime_diagnostics_provider=runtime_health.inspect, @@ -984,7 +1027,6 @@ def authorize_bridge_workspace( edit_bridge_service=edit_bridge_service, services=( WorkspaceBootstrapLifecycle(tuple(workspaces.values())), - _OptionalNativeRuntimeLifecycle(capability_statuses), writer, recall, runtime_health, diff --git a/daem0nmcp/capabilities.py b/daem0nmcp/capabilities.py index 6b17e0e..71e8a5e 100644 --- a/daem0nmcp/capabilities.py +++ b/daem0nmcp/capabilities.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from importlib import metadata -from typing import Any +from typing import Any, Literal CAPABILITY_STATUSES = frozenset({"ready", "disabled", "degraded", "failed"}) CORE_DISTRIBUTIONS = ( @@ -117,10 +117,15 @@ def get(self, name: str) -> dict[str, Any]: configured = self._configured(profile) if configured is None: return self._failed(profile) - if not configured: + if configured is False: return self._disabled(profile) + # An unset variable means "on when installed": a profile that cannot run + # is simply off, while an explicit request for it is degraded. if name == "models-local" and sys.version_info < (3, 11): - return self._python_version_unavailable(profile) + capability = self._python_version_unavailable(profile) + if configured == "auto": + capability["status"] = "disabled" + return capability missing = [ distribution @@ -128,6 +133,8 @@ def get(self, name: str) -> dict[str, Any]: if not self._distribution_available(distribution) ] if missing: + if configured == "auto": + return self._not_installed(profile, missing) return self._degraded(profile, missing) return self._ready(profile) @@ -167,11 +174,13 @@ def _core(self) -> dict[str, Any]: }, } - def _configured(self, profile: CapabilityProfile) -> bool | None: - value = self._environ.get(profile.environment_key, "false").strip().lower() + def _configured(self, profile: CapabilityProfile) -> bool | Literal["auto"] | None: + value = self._environ.get(profile.environment_key, "").strip().lower() + if not value: + return "auto" if value in {"1", "true", "yes", "on"}: return True - if value in {"0", "false", "no", "off", ""}: + if value in {"0", "false", "no", "off"}: return False return None @@ -191,6 +200,24 @@ def _disabled(profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "enable_configuration", "environment": f"{profile.environment_key}=true", + "message": ( + f"Unset {profile.environment_key} or set it to true; the " + f"profile turns on when 'daem0nmcp[{profile.name}]' is installed." + ), + }, + } + + @staticmethod + def _not_installed( + profile: CapabilityProfile, missing: list[str] + ) -> dict[str, Any]: + return { + "name": profile.name, + "status": "disabled", + "remediation": { + "action": "install_extra", + "command": f"pip install 'daem0nmcp[{profile.name}]'", + "missing": missing, }, } @@ -225,7 +252,10 @@ def _failed(self, profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "fix_configuration", "environment": profile.environment_key, - "message": "Use a boolean value: true or false.", + "message": ( + "Use true or false, or leave it unset to enable the " + "profile automatically when its extra is installed." + ), }, } diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index ed157ed..6038d97 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -1247,15 +1247,18 @@ async def run_watcher(): if args.json: print( json.dumps( - {"error": "tree-sitter-languages not installed", "indexed": 0} + { + "error": "tree-sitter-language-pack not installed", + "indexed": 0, + } ) ) else: print( - "ERROR: Code indexing requires tree-sitter-languages", + "ERROR: Code indexing requires tree-sitter-language-pack", file=sys.stderr, ) - print("Install with: pip install tree-sitter-languages") + print("Install with: pip install 'daem0nmcp[apps]'") sys.exit(1) project_path = Path(args.project_path or os.getcwd()).resolve() diff --git a/daem0nmcp/opencode_assets/commands/commune.md b/daem0nmcp/opencode_assets/commands/commune.md new file mode 100644 index 0000000..27a2781 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/commune.md @@ -0,0 +1,13 @@ +--- +description: Start the Daem0n v7 session for a registered workspace +--- + +Call `daem0nmcp_session_brief(workspace_id="$ARGUMENTS")` immediately. The +argument must be the configured opaque `workspace_id`, not a path. + +After receiving the briefing results: +1. Report the session status, active warnings, and recent activity summary +2. Acknowledge that the v7 scoped session is active + +This slash-command filename is only a host shortcut. For v6 migration details, +use `docs/v6-to-v7-tools.json`. diff --git a/daem0nmcp/opencode_assets/commands/counsel.md b/daem0nmcp/opencode_assets/commands/counsel.md new file mode 100644 index 0000000..d8cd22f --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/counsel.md @@ -0,0 +1,24 @@ +--- +description: Request an exact Daem0n v7 preflight capability +--- + +Parse `$ARGUMENTS` as the intended protected v7 tool and its arguments, then +call: + +```text +daem0nmcp_memory_preflight( + workspace_id="", + target_tool="", + target_arguments={}, + description="$ARGUMENTS" +) +``` + +After receiving the preflight results: +1. Report any WARNINGS the user must know about +2. Report any FAILED APPROACHES that are relevant +3. Report any must_not constraints that apply +4. Preserve the token only for the exact request it authorizes + +Do not claim a generic edit authorization. The server binds the capability to +the workspace, principal, session, target tool, and canonical arguments. diff --git a/daem0nmcp/opencode_assets/commands/inscribe.md b/daem0nmcp/opencode_assets/commands/inscribe.md new file mode 100644 index 0000000..1359bb2 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/inscribe.md @@ -0,0 +1,31 @@ +--- +description: Store a replay-safe Daem0n v7 memory record +--- + +Store the following as a v7 memory record: + +$ARGUMENTS + +Choose `record_type` from the content: +- "decision" for architectural or design choices +- "pattern" for recurring approaches to follow +- "warning" for things to avoid +- "learning" for lessons from experience + +Create one stable idempotency key. Call `daem0nmcp_memory_preflight` with +`target_tool="memory_store"` and the exact target arguments, then call: + +```text +daem0nmcp_memory_store( + workspace_id="", + record_type="", + content="$ARGUMENTS", + rationale="", + idempotency_key="", + preflight_token="" +) +``` + +Report the returned `record_id`. After verification, use +`daem0nmcp_memory_record_outcome` with that ID and a separate stable +idempotency key. diff --git a/daem0nmcp/opencode_assets/commands/recall.md b/daem0nmcp/opencode_assets/commands/recall.md new file mode 100644 index 0000000..e372a59 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/recall.md @@ -0,0 +1,14 @@ +--- +description: Retrieve bounded memories from a Daem0n v7 workspace +--- + +Call `daem0nmcp_memory_recall(workspace_id="", query="$ARGUMENTS", limit=10)` immediately. + +Present the retrieved memories in a clear format: +- Show each record's content, type, and creation time +- Highlight any warnings or failed approaches +- Preserve evidence references and truncation indicators + +For service diagnostics use +`daem0nmcp_system_health(workspace_id="")`. Do not replace the +opaque workspace ID with a project path. diff --git a/daem0nmcp/opencode_install.py b/daem0nmcp/opencode_install.py index bddb83a..98f05b5 100644 --- a/daem0nmcp/opencode_install.py +++ b/daem0nmcp/opencode_install.py @@ -1,8 +1,8 @@ """Installer for OpenCode integration. -Creates .opencode/ directory structure, ensures opencode.json exists -at project root, and writes the TypeScript covenant enforcement plugin -for MCP server connectivity and hook discipline. +Creates the .opencode/ directory structure and slash-command files, ensures +opencode.json exists at project root, and writes the TypeScript covenant +enforcement plugin for MCP server connectivity and hook discipline. """ import argparse @@ -44,12 +44,14 @@ def select_opencode_interface(version: str, requested: str | None = None) -> str # Subdirectories to scaffold inside .opencode/ _OPENCODE_SUBDIRS = ["commands", "plugins", "agents"] -# The packaged TypeScript resource is the sole installer source of truth. -PLUGIN_TEMPLATE = ( - importlib.resources.files("daem0nmcp.opencode_assets") - .joinpath("daem0n.ts") - .read_text(encoding="utf-8") -) +# The packaged resources are the sole installer source of truth. +_ASSETS = importlib.resources.files("daem0nmcp.opencode_assets") +PLUGIN_TEMPLATE = _ASSETS.joinpath("daem0n.ts").read_text(encoding="utf-8") +COMMAND_TEMPLATES = { + command.name: command.read_text(encoding="utf-8") + for command in _ASSETS.joinpath("commands").iterdir() + if command.name.endswith(".md") +} def detect_clients(project_path: Path) -> dict[str, Any]: @@ -96,20 +98,45 @@ def _ensure_dir(path: Path, dry_run: bool) -> str: return "[create]" -def _ensure_file(path: Path, content: str, dry_run: bool, force: bool) -> str: +def _reject_links(path: Path, root: Path) -> None: + """Refuse a target that a link could move outside the project.""" + current = path + while True: + if current.is_symlink(): + raise OSError(f"refusing to write through the link {current}") + if current == root: + return + parent = current.parent + if parent == current: + raise OSError(f"{path} is outside {root}") + current = parent + + +def _ensure_file( + path: Path, content: str, dry_run: bool, force: bool, root: Path +) -> str: """Ensure a file exists with given content. Returns status string.""" + _reject_links(path, root) if path.exists(): if force: if not dry_run: - path.write_text(content, encoding="utf-8") + # Replace rather than write in place, so the new file is ours. + path.unlink() + _write_new(path, content) return "[overwrite]" return "[exists]" if not dry_run: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") + _reject_links(path.parent, root) + _write_new(path, content) return "[create]" +def _write_new(path: Path, content: str) -> None: + with path.open("x", encoding="utf-8") as handle: + handle.write(content) + + def _configured_template( installation: BridgeInstallation | None, ) -> dict[str, Any]: @@ -266,7 +293,7 @@ def install_opencode( lines.append("Configuration:") json_content = json.dumps(_configured_template(installation), indent=2) + "\n" json_path = root / "opencode.json" - status = _ensure_file(json_path, json_content, dry_run, force) + status = _ensure_file(json_path, json_content, dry_run, force, root) lines.append(f" {status} opencode.json") if ( installation is not None @@ -294,9 +321,16 @@ def install_opencode( # -- Ensure plugin file ------------------------------------------ plugin_path = opencode_root / "plugins" / "daem0n.ts" - status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force) + status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force, root) lines.append(f" {status} .opencode/plugins/daem0n.ts") + # -- Ensure slash-command files ---------------------------------- + for name, content in sorted(COMMAND_TEMPLATES.items()): + status = _ensure_file( + opencode_root / "commands" / name, content, dry_run, force, root + ) + lines.append(f" {status} .opencode/commands/{name}") + # -- AGENTS.md status (do NOT create) ---------------------------- if oc["agents_md"]: lines.append(" [exists] AGENTS.md") diff --git a/pyproject.toml b/pyproject.toml index 09a50ed..e906eb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ include = ["daem0nmcp*"] "static/*.js", "static/renderers/*.js", ] -"daem0nmcp.opencode_assets" = ["*.ts"] +"daem0nmcp.opencode_assets" = ["*.ts", "commands/*.md"] # Ruff configuration [tool.ruff] diff --git a/start_daem0nmcp_server.bat b/start_daem0nmcp_server.bat index fb71a2c..08214d1 100644 --- a/start_daem0nmcp_server.bat +++ b/start_daem0nmcp_server.bat @@ -3,13 +3,26 @@ REM ============================================ REM Daem0nMCP HTTP Server Launcher for Windows REM ============================================ REM This script starts the Daem0nMCP HTTP server -REM for use with Claude Code on Windows. +REM for one project on Windows. REM -REM Windows has known issues with stdio transport, -REM so HTTP transport is required. +REM Usage: start_daem0nmcp_server.bat [project_dir] +REM The project defaults to an exported +REM DAEM0NMCP_PROJECT_ROOT, else the directory you +REM run it from. Stdio (python -m daem0nmcp) also +REM works on Windows; use this launcher when you +REM want an HTTP server instead. REM ============================================ title Daem0nMCP Server +REM Delayed expansion would re-parse ! and ^ in a project path +setlocal DisableDelayedExpansion + +REM Capture the project before changing directory +set "DAEM0N_PROJECT=" +if not "%~1"=="" set "DAEM0N_PROJECT=%~f1" +if "%~1"=="" if not defined DAEM0NMCP_PROJECT_ROOT set "DAEM0N_PROJECT=%CD%" +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%" goto :system_directory +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%\System32" goto :system_directory REM Change to the script's directory cd /d "%~dp0" @@ -25,8 +38,13 @@ echo \^|/ echo * echo. -REM Start the server -python start_server.py --port 9876 +REM Start the server. An exported DAEM0NMCP_PROJECT_ROOT is left alone. +REM The trailing "\." keeps a drive root such as C:\ from escaping the quote +if defined DAEM0N_PROJECT ( + python start_server.py --port 9876 --project "%DAEM0N_PROJECT%\." +) else ( + python start_server.py --port 9876 +) REM If the server exits, pause so user can see any errors if errorlevel 1 ( @@ -34,3 +52,12 @@ if errorlevel 1 ( echo [ERROR] Server exited with an error. Press any key to close. pause >nul ) +goto :eof + +:system_directory +echo. +echo [ERROR] Refusing to serve "%DAEM0N_PROJECT%". +echo Run the launcher from your project, or pass it: +echo start_daem0nmcp_server.bat C:\path\to\project +pause >nul +exit /b 2 diff --git a/start_server.py b/start_server.py index ef3ffb7..dd5f5cf 100644 --- a/start_server.py +++ b/start_server.py @@ -22,12 +22,22 @@ def main(): parser.add_argument("--host", default="127.0.0.1", help="Host to bind to") parser.add_argument( "--project", - help="Project directory for storage (default: current directory)", + help=( + "Project directory to serve (default: DAEM0NMCP_PROJECT_ROOT if set, " + "else the current directory)" + ), ) args = parser.parse_args() - project_root = Path(args.project or os.getcwd()).resolve() - os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root) + project_root = Path( + args.project or os.environ.get("DAEM0NMCP_PROJECT_ROOT") or Path.cwd() + ) + if not project_root.is_dir(): + # Otherwise the server silently creates and serves an empty workspace + # wherever a typo points. + parser.error(f"project directory does not exist: {project_root}") + os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root.resolve()) + print(f"Serving project: {project_root.resolve()}", file=sys.stderr) # Import only after the workspace environment is fixed. Both public # launchers use the same v7 composition and transport-security boundary. diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index e31e2f9..4dca20a 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -150,6 +150,29 @@ def server_environment( return environment +_LOG_TAIL_BYTES = 8000 + + +def dump_server_log(path: Path) -> None: + """Print the server's own log when a process test fails. + + CI shows only the assertion, so an INTERNAL_ERROR or a crash inside the + server is otherwise invisible; the tmp directory is gone by then. + """ + + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError as error: + print(f"\n----- {path.name} unavailable: {error} -----", flush=True) + return + tail = text[-_LOG_TAIL_BYTES:] + print( + f"\n----- {path.name} (last {len(tail)} of {len(text)} chars) -----\n{tail}" + f"\n----- end {path.name} -----", + flush=True, + ) + + @asynccontextmanager async def process_client( workspace: Path, @@ -192,10 +215,12 @@ async def process_client( # SDK stdout-reader BrokenResourceError during shutdown. body_error = error except BaseException as cleanup_error: + dump_server_log(workspace / log_name) if body_error is not None: raise body_error from cleanup_error raise if body_error is not None: + dump_server_log(workspace / log_name) raise body_error return if transport != "streamable-http": @@ -203,68 +228,79 @@ async def process_client( with socket.socket() as reservation: reservation.bind(("127.0.0.1", 0)) port = reservation.getsockname()[1] - with (workspace / "http-server.log").open("w", encoding="utf-8") as log: - process = subprocess.Popen( - [ - python, - "-m", - "daem0nmcp.server", - "--transport", - transport, - "--host", - "127.0.0.1", - "--port", - str(port), - ], - cwd=workspace, - env=environment, - stdout=log, - stderr=log, - creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0, - ) - try: - deadline = asyncio.get_running_loop().time() + 30 - while True: - if process.poll() is not None: - raise RuntimeError( - f"MCP server exited with code {process.returncode}; inspect http-server.log" - ) - try: - _, writer = await asyncio.open_connection("127.0.0.1", port) - writer.close() - await writer.wait_closed() - break - except OSError: - if asyncio.get_running_loop().time() >= deadline: - raise TimeoutError( - "MCP HTTP server did not become ready" - ) from None - await asyncio.sleep(0.05) - if http_probe is not None: - await http_probe(f"http://127.0.0.1:{port}/mcp") - async with ( - httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, - streamable_http_client( - f"http://127.0.0.1:{port}/mcp", http_client=http_client - ) as ( - read, - write, - _, - ), - ClientSession( - read, write, read_timeout_seconds=timedelta(seconds=30) - ) as session, - ): - await session.initialize() - yield session - finally: - if process.poll() is None: - process.terminate() + failed = False + try: + with (workspace / "http-server.log").open("w", encoding="utf-8") as log: + process = subprocess.Popen( + [ + python, + "-m", + "daem0nmcp.server", + "--transport", + transport, + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=workspace, + env=environment, + stdout=log, + stderr=log, + creationflags=subprocess.CREATE_NO_WINDOW + if sys.platform == "win32" + else 0, + ) try: - await asyncio.to_thread(process.wait, timeout=10) - except subprocess.TimeoutExpired: - process.kill() - await asyncio.to_thread(process.wait, timeout=10) + deadline = asyncio.get_running_loop().time() + 30 + while True: + if process.poll() is not None: + raise RuntimeError( + f"MCP server exited with code {process.returncode}; inspect http-server.log" + ) + try: + _, writer = await asyncio.open_connection("127.0.0.1", port) + writer.close() + await writer.wait_closed() + break + except OSError: + if asyncio.get_running_loop().time() >= deadline: + raise TimeoutError( + "MCP HTTP server did not become ready" + ) from None + await asyncio.sleep(0.05) + if http_probe is not None: + await http_probe(f"http://127.0.0.1:{port}/mcp") + async with ( + httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, + streamable_http_client( + f"http://127.0.0.1:{port}/mcp", http_client=http_client + ) as ( + read, + write, + _, + ), + ClientSession( + read, write, read_timeout_seconds=timedelta(seconds=30) + ) as session, + ): + await session.initialize() + yield session + except BaseException: + failed = True + raise + finally: + if process.poll() is None: + process.terminate() + try: + await asyncio.to_thread(process.wait, timeout=10) + except subprocess.TimeoutExpired: + process.kill() + await asyncio.to_thread(process.wait, timeout=10) + finally: + if failed: + # Outside the log context, so the tail is flushed and complete. + dump_server_log(workspace / "http-server.log") async def call(session: ClientSession, tool: str, arguments: dict) -> dict: diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index eab8404..452ccc8 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1665,6 +1665,30 @@ def factory(): self.assertFalse(loaded) self.assertEqual("CAPABILITY_DISABLED", raised.exception.code) self.assertEqual("apps", raised.exception.capability_states[0].name) + self.assertIn( + "daem0nmcp[apps]", raised.exception.capability_states[0].remediation + ) + + async def test_each_apps_status_names_its_own_remedy(self) -> None: + """An invalid setting is a configuration fix, not a missing package.""" + from daem0nmcp.api.v7.discovery_operations import _code_capability_error + + remedies = {} + for status in ("disabled", "degraded", "failed"): + error = _code_capability_error( + SimpleNamespace(capability_statuses={"apps": status}) + ) + assert error is not None + remedies[status] = error.capability_states[0].remediation + + self.assertIn("unless DAEM0NMCP_APPS_ENABLED=false", remedies["disabled"]) + self.assertEqual(remedies["degraded"], remedies["disabled"]) + self.assertIn("true, false, or leave it unset", remedies["failed"]) + self.assertIsNone( + _code_capability_error( + SimpleNamespace(capability_statuses={"apps": "ready"}) + ) + ) async def test_code_index_source_change_rolls_back_before_publish(self) -> None: from daem0nmcp.api.v7.discovery_operations import ( diff --git a/tests/api_v7/test_entrypoints.py b/tests/api_v7/test_entrypoints.py index dcd0b15..fdad6c8 100644 --- a/tests/api_v7/test_entrypoints.py +++ b/tests/api_v7/test_entrypoints.py @@ -2,7 +2,9 @@ import io import os +import shutil import sys +import tempfile import types import unittest from contextlib import redirect_stdout @@ -15,6 +17,9 @@ def test_http_launcher_builds_remote_v7_server_and_uses_shared_runner(self) -> N import start_server from daem0nmcp.api.v7.launcher import ServerOptions + workspace = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, workspace, True) + built: list[tuple[str, str | None]] = [] launched: list[tuple[object, ServerOptions]] = [] server = object() @@ -26,7 +31,7 @@ def create_server( ) -> object: self.assertEqual( os.environ.get("DAEM0NMCP_PROJECT_ROOT"), - str(Path("tests/http-workspace").resolve()), + str(workspace.resolve()), ) built.append((transport, host)) return server @@ -40,7 +45,7 @@ def create_server( "--port", "9988", "--project", - str(Path("tests/http-workspace")), + str(workspace), ] output = io.StringIO() with ( diff --git a/tests/api_v7/test_external_operations.py b/tests/api_v7/test_external_operations.py index 1a1126d..eda7561 100644 --- a/tests/api_v7/test_external_operations.py +++ b/tests/api_v7/test_external_operations.py @@ -737,6 +737,46 @@ async def test_missing_api_key_keeps_operation_registered_but_disabled( "Set E2B_API_KEY to enable isolated Python execution.", ) + async def test_disabling_the_profile_switches_remote_execution_off(self) -> None: + """The advertised opt-out must also stop the sandbox.""" + from daem0nmcp.api.v7.external_operations import ( + ExternalOperationDependencies, + ExternalOperationError, + build_external_operations, + ) + from daem0nmcp.api.v7.record_operations import RecordOperationDependencies + from daem0nmcp.workspace import WorkspaceRegistry + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + workspace = WorkspaceRegistry([root], default_root=root).default + operations = build_external_operations( + ExternalOperationDependencies( + record_dependencies=RecordOperationDependencies(), + environment={ + "E2B_API_KEY": "key", + "DAEM0NMCP_AGENCY_E2B_ENABLED": "false", + }, + sandbox_provider_factory=lambda: self.fail("provider was built"), + ) + ) + with self.assertRaises(ExternalOperationError) as caught: + await operations["sandbox_execute_python"]( + workspace=workspace, + request=_request( + "sandbox_execute_python", + workspace_id=workspace.workspace_id, + code="print(1)", + timeout_seconds=1, + preflight_token=PREFLIGHT_TOKEN, + ), + ) + + self.assertEqual(caught.exception.code, "CAPABILITY_DISABLED") + state = caught.exception.capability_states[0] + self.assertEqual(state.name, "agency-e2b") + self.assertIn("DAEM0NMCP_AGENCY_E2B_ENABLED", str(state.remediation)) + if __name__ == "__main__": unittest.main() diff --git a/tests/api_v7/test_graph_operations.py b/tests/api_v7/test_graph_operations.py index 12befa2..a8b7336 100644 --- a/tests/api_v7/test_graph_operations.py +++ b/tests/api_v7/test_graph_operations.py @@ -842,7 +842,12 @@ async def test_actual_mcp_graph_profile_enabled_and_disabled(tmp_path, transport "force": False, "idempotency_key": "process-graph-disabled", } - async with process_client(workspace.root, transport) as session: + # The graph extra is installed in CI, so it is on unless explicitly disabled. + async with process_client( + workspace.root, + transport, + environment_overrides={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) token = await _preflight( session, workspace.workspace_id, "entity_backfill", disabled_arguments diff --git a/tests/api_v7/test_portable_projections.py b/tests/api_v7/test_portable_projections.py index 0633ffc..417d12d 100644 --- a/tests/api_v7/test_portable_projections.py +++ b/tests/api_v7/test_portable_projections.py @@ -1995,3 +1995,24 @@ def blocking_prepare(*args, **kwargs): assert artifacts == 0 attempts = directory / "attempts" assert not attempts.exists() or not any(attempts.iterdir()) + + +def test_a_concurrent_writer_is_reported_as_retryable_not_as_a_bad_bundle(): + """A held SQLite write lock must not read as an invalid bundle.""" + from daem0nmcp.api.v7.operations import _portable_failure_code + from daem0nmcp.api.v7.portable_projections import PortableTransferError + + busy = sqlite3.OperationalError("database is locked") + wrapped = PortableTransferError("IMPORT_INVALID") + wrapped.__cause__ = busy + # A bad bundle raised while an unrelated busy error is in flight. + unrelated = PortableTransferError("IMPORT_INVALID") + unrelated.__context__ = busy + + assert _portable_failure_code(busy, "workspace_import") == "DATABASE_IN_USE" + assert _portable_failure_code(wrapped, "workspace_import") == "DATABASE_IN_USE" + assert _portable_failure_code(unrelated, "workspace_import") == "IMPORT_INVALID" + assert ( + _portable_failure_code(ValueError("bad page"), "workspace_import") + == "IMPORT_INVALID" + ) diff --git a/tests/api_v7/test_process_client.py b/tests/api_v7/test_process_client.py index 8f71340..98cad9f 100644 --- a/tests/api_v7/test_process_client.py +++ b/tests/api_v7/test_process_client.py @@ -452,3 +452,22 @@ async def test_consolidation_preview_copy_and_canonical_archive_over_real_mcp( ) assert archived["archived"] == 2 assert all(source.root.is_dir() for source in sources) + + +async def test_an_installed_extra_is_enabled_without_any_variable( + initialized_workspace, +): + """UD-7: installing the extra is the whole opt-in.""" + from daem0nmcp.capabilities import CapabilityRegistry + + if CapabilityRegistry(environ={}).get("graph")["status"] != "ready": + pytest.skip("the graph extra is not installed") + + workspace = initialized_workspace + scope = {"workspace_id": workspace.workspace_id} + async with process_client(workspace.root, "stdio") as session: + await succeed(session, "session_brief", scope) + health = await succeed(session, "system_health", scope) + + states = {state["name"]: state["status"] for state in health["capability_states"]} + assert states["graph"] == "ready", states diff --git a/tests/api_v7/test_process_code_impact.py b/tests/api_v7/test_process_code_impact.py index 4beb9dd..8e7f852 100644 --- a/tests/api_v7/test_process_code_impact.py +++ b/tests/api_v7/test_process_code_impact.py @@ -111,7 +111,11 @@ async def test_production_code_index_search_and_impact(code_workspace, transport async def test_production_code_index_disabled_profile_has_remedy(code_workspace): scope = {"workspace_id": code_workspace.workspace_id} - async with process_client(code_workspace.root, "stdio") as session: + async with process_client( + code_workspace.root, + "stdio", + environment_overrides={"DAEM0NMCP_APPS_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) disabled = await call( session, @@ -122,4 +126,4 @@ async def test_production_code_index_disabled_profile_has_remedy(code_workspace) assert disabled["error"]["code"] == "CAPABILITY_DISABLED" states = disabled["meta"]["capability_states"] assert states[0]["name"] == "apps" - assert "DAEM0NMCP_APPS_ENABLED=true" in states[0]["remediation"] + assert "unless DAEM0NMCP_APPS_ENABLED=false" in states[0]["remediation"] diff --git a/tests/api_v7/test_process_dreaming.py b/tests/api_v7/test_process_dreaming.py index 8954f36..bc391a6 100644 --- a/tests/api_v7/test_process_dreaming.py +++ b/tests/api_v7/test_process_dreaming.py @@ -82,6 +82,8 @@ async def test_production_dreaming_runs_idempotently_and_reports_missing_graph( "DAEM0NMCP_DREAM_PENDING_MIN_AGE_HOURS": "0", "DAEM0NMCP_DREAM_PENDING_EVIDENCE_THRESHOLD": "1", "DAEM0NMCP_DREAM_REVIEW_COOLDOWN_HOURS": "72", + # CI installs the graph extra, which would otherwise turn graph on. + "DAEM0NMCP_GRAPH_ENABLED": "false", } scope = {"workspace_id": workspace.workspace_id} async with process_client( diff --git a/tests/api_v7/test_process_portability.py b/tests/api_v7/test_process_portability.py index 40e135d..f86cd8c 100644 --- a/tests/api_v7/test_process_portability.py +++ b/tests/api_v7/test_process_portability.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import sqlite3 from contextlib import closing from datetime import datetime, timedelta, timezone @@ -12,7 +13,7 @@ from daem0nmcp.database import DatabaseManager from daem0nmcp.event_store import EventCommand, EventStore from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed from tests.api_v7.test_process_surface import _preflight, _store @@ -40,6 +41,52 @@ async def _export_pages(session, scope, legacy): return pages +async def _import_retrying_while_busy(session, scope, arguments): + """Import one page, waiting out a concurrent writer's SQLite write lock. + + A projection drain or dreaming job can hold the write lock past the server's + busy timeout, which both the preflight and the transfer report as the + retryable DATABASE_IN_USE. A real client retries those; so does this test, + within a bounded deadline and a bounded number of attempts. + """ + + deadline = asyncio.get_running_loop().time() + 30 + for attempt in range(8): + contended = None + preflight = await call( + session, + "memory_preflight", + { + **scope, + "target_tool": "workspace_import", + "target_arguments": arguments, + "description": "Exercise workspace_import through the transport", + }, + ) + if preflight["ok"]: + result = await call( + session, + "workspace_import", + { + **scope, + **arguments, + "preflight_token": preflight["data"]["preflight_token"], + }, + ) + if result["ok"]: + # Contention is occasional; a transfer that always contends is + # a regression, not slowness. + assert attempt < 4, f"import needed {attempt + 1} attempts" + return result["data"] + contended = result["error"] + else: + contended = preflight["error"] + assert contended["code"] == "DATABASE_IN_USE", contended + assert asyncio.get_running_loop().time() < deadline, contended + await asyncio.sleep(0.5) + raise AssertionError("the import stayed contended for 8 attempts") + + @pytest.mark.parametrize("transport", ["stdio", "streamable-http"]) @pytest.mark.parametrize("legacy", [False, True]) async def test_production_export_restores_retained_workspace( @@ -78,7 +125,19 @@ async def test_production_export_restores_retained_workspace( retained = tmp_path / ".daem0nmcp" / "retained-before-restore" assert storage.resolve().is_relative_to(tmp_path.resolve()) assert retained.resolve().is_relative_to(tmp_path.resolve()) - storage.rename(retained) + # Windows can briefly refuse the rename after the server exits. The cause is + # unconfirmed (the venv launcher's real interpreter is a grandchild the + # harness does not wait for, or a virus scanner reading the fresh files); + # no product code renames storage. Retry within a bounded deadline. + rename_deadline = asyncio.get_running_loop().time() + 15 + while True: + try: + storage.rename(retained) + break + except PermissionError: + if asyncio.get_running_loop().time() >= rename_deadline: + raise + await asyncio.sleep(0.2) async with process_client(tmp_path, transport) as session: await succeed(session, "session_brief", scope) import_id = None @@ -90,17 +149,7 @@ async def test_production_export_restores_retained_workspace( } if import_id is not None: arguments["import_session_id"] = import_id - staged = await succeed( - session, - "workspace_import", - { - **scope, - **arguments, - "preflight_token": await _preflight( - session, scope, "workspace_import", arguments - ), - }, - ) + staged = await _import_retrying_while_busy(session, scope, arguments) import_id = staged["import_session_id"] assert staged["staged_pages"] == index + 1 assert staged["imported"] == 0 @@ -109,17 +158,7 @@ async def test_production_export_restores_retained_workspace( "finalize": True, "idempotency_key": "portable-process-import-0001", } - restored = await succeed( - session, - "workspace_import", - { - **scope, - **finish, - "preflight_token": await _preflight( - session, scope, "workspace_import", finish - ), - }, - ) + restored = await _import_retrying_while_busy(session, scope, finish) assert restored["status"] == "succeeded" assert restored["imported"] == 2 assert restored["root_hash"] == expected_root diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index 7bfc48b..d878d19 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -98,7 +98,7 @@ def test_native_runtime_preloads_only_each_enabled_profile( "sentence_transformers", "onnx", "onnxruntime", - "numpy", + # numpy belongs to models-local; the graph extra never installs it. "networkx", "igraph", "leidenalg", @@ -106,6 +106,46 @@ def test_native_runtime_preloads_only_each_enabled_profile( imported, ) + def test_a_broken_installed_runtime_degrades_its_own_profile_only(self) -> None: + """Metadata can say ready while the package cannot import here.""" + from daem0nmcp.api.v7 import production + + def import_module(name: str) -> object: + if name == "igraph": + raise OSError("[WinError 126] The specified module could not be found") + return object() + + statuses = {"graph": "ready", "local": "ready"} + with patch.object(production.importlib, "import_module", import_module): + lifecycle = production._OptionalNativeRuntimeLifecycle(statuses) + lifecycle.start() + + self.assertEqual({"graph": "degraded", "local": "ready"}, statuses) + self.assertIn("igraph", lifecycle.failures["graph"]) + self.assertIn("OSError", lifecycle.failures["graph"]) + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "true"}, lifecycle.failures + ) + } + self.assertEqual("degraded", states["graph"].status) + self.assertIn("igraph", str(states["graph"].remediation)) + + def test_capability_states_carry_the_registry_remediation(self) -> None: + """Health must say how to enable a profile, not 'review' it.""" + from daem0nmcp.api.v7 import production + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "false"} + ) + } + self.assertEqual("disabled", states["graph"].status) + self.assertIn("DAEM0NMCP_GRAPH_ENABLED", str(states["graph"].remediation)) + def test_full_manifest_rejects_missing_or_unexpected_resources(self) -> None: from daem0nmcp.api.v7.production import build_production_surface from daem0nmcp.api.v7.registry import ManifestError diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 8459d84..ef5712c 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -218,6 +218,95 @@ def test_disabled_capability_does_not_probe_its_optional_modules(self): "DAEM0NMCP_MODELS_LOCAL_ENABLED=true", ) + def test_unset_variable_enables_an_installed_profile(self): + """An installed extra turns its profile on without any variable.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("graph") + + self.assertEqual(capability["status"], "ready") + + def test_unset_variable_leaves_a_missing_extra_disabled(self): + """Without the extra, an unrequested profile is off, not degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda name: name != "leidenalg" + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual( + capability["remediation"], + { + "action": "install_extra", + "command": "pip install 'daem0nmcp[graph]'", + "missing": ["leidenalg"], + }, + ) + + def test_an_empty_value_is_treated_as_unset(self): + """An empty variable means auto, as an unset one does.""" + from daem0nmcp.capabilities import CapabilityRegistry + + installed = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": " "}, + module_available=lambda _: True, + ).get("graph") + missing = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": ""}, + module_available=lambda _: False, + ).get("graph") + + self.assertEqual(installed["status"], "ready") + self.assertEqual(missing["status"], "disabled") + self.assertEqual(missing["remediation"]["action"], "install_extra") + + def test_explicit_false_disables_an_installed_profile(self): + """The variable still turns an installed profile off, without probing.""" + from daem0nmcp.capabilities import CapabilityRegistry + + def probe(distribution): + raise AssertionError(f"probed {distribution} for a disabled profile") + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + module_available=probe, + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "enable_configuration") + self.assertIn( + "Unset DAEM0NMCP_GRAPH_ENABLED or set it to true", + capability["remediation"]["message"], + ) + + def test_explicit_true_with_a_missing_extra_is_degraded(self): + """Asking for a profile whose packages are absent is a degraded request.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "true"}, + module_available=lambda name: name != "leidenalg", + ).get("graph") + + self.assertEqual(capability["status"], "degraded") + self.assertEqual(capability["remediation"]["action"], "install_extra") + self.assertEqual(capability["remediation"]["missing"], ["leidenalg"]) + + def test_unset_models_local_is_disabled_on_unsupported_python(self): + """Auto mode never reports an unrequested profile as degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + with patch("daem0nmcp.capabilities.sys.version_info", (3, 10, 21)): + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("models-local") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "upgrade_python") + def test_invalid_enablement_value_is_failed_with_structured_remediation(self): """An invalid profile setting is a configuration failure, not a missing extra.""" from daem0nmcp.capabilities import CapabilityRegistry @@ -234,6 +323,7 @@ def test_invalid_enablement_value_is_failed_with_structured_remediation(self): self.assertEqual( capability["remediation"]["environment"], "DAEM0NMCP_MODELS_LOCAL_ENABLED" ) + self.assertIn("unset", capability["remediation"]["message"]) def test_ready_profile_checks_only_the_requested_profile(self): """Capability inspection must not import or probe every optional subsystem.""" diff --git a/tests/test_opencode_install.py b/tests/test_opencode_install.py index a94b63a..8b0c0c6 100644 --- a/tests/test_opencode_install.py +++ b/tests/test_opencode_install.py @@ -11,6 +11,7 @@ from daem0nmcp.edit_bridge_transport import provision_bridge_credential from daem0nmcp.opencode_install import ( + COMMAND_TEMPLATES, PLUGIN_TEMPLATE, install_opencode, select_opencode_interface, @@ -228,3 +229,80 @@ def test_install_preserves_existing_opencode_json(tmp_path): assert plugin.exists(), ( "Plugin file should be created even when opencode.json exists" ) + + +def test_install_creates_repo_command_files_without_overwriting(tmp_path): + """The installer ships the repo's slash commands and keeps user edits.""" + expected = _repo_commands() + assert set(expected) == {"commune.md", "counsel.md", "inscribe.md", "recall.md"} + commands = tmp_path / ".opencode" / "commands" + commands.mkdir(parents=True) + (commands / "recall.md").write_text("my recall", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[exists] .opencode/commands/recall.md" in msg + assert "[create] .opencode/commands/commune.md" in msg + assert (commands / "recall.md").read_text(encoding="utf-8") == "my recall" + for name in ("commune.md", "counsel.md", "inscribe.md"): + assert (commands / name).read_text(encoding="utf-8") == expected[name] + + +def _repo_commands() -> dict[str, str]: + repo_commands = Path(__file__).resolve().parents[1] / ".opencode" / "commands" + return { + path.name: path.read_text(encoding="utf-8") + for path in repo_commands.glob("*.md") + } + + +def test_packaged_commands_match_the_repo_commands(): + """The shipped copies must not drift from the ones the repo uses.""" + assert _repo_commands() == COMMAND_TEMPLATES + + +def test_force_overwrites_commands_and_dry_run_writes_nothing(tmp_path): + ok, msg = install_opencode( + str(tmp_path), dry_run=True, bridge_config_root=tmp_path / "host-config" + ) + assert ok, msg + assert "[create] .opencode/commands/recall.md" in msg + assert not (tmp_path / ".opencode" / "commands" / "recall.md").exists() + + assert install_opencode(str(tmp_path), bridge_config_root=tmp_path / "host-config")[ + 0 + ] + recall = tmp_path / ".opencode" / "commands" / "recall.md" + recall.write_text("edited", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), force=True, bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[overwrite] .opencode/commands/recall.md" in msg + assert recall.read_text(encoding="utf-8") == COMMAND_TEMPLATES["recall.md"] + + +def test_installer_refuses_to_write_through_a_symlink(tmp_path): + """A cloned repo must not steer the installer outside the project.""" + outside = tmp_path / "outside" + outside.mkdir() + project = tmp_path / "project" + (project / ".opencode" / "commands").mkdir(parents=True) + link = project / ".opencode" / "commands" / "recall.md" + try: + link.symlink_to(outside / "stolen.md") + except (OSError, NotImplementedError): + pytest.skip("symlinks are unavailable") + + ok, msg = install_opencode( + str(project), bridge_config_root=tmp_path / "host-config" + ) + + assert not ok + assert "link" in msg + assert not (outside / "stolen.md").exists() diff --git a/tests/test_start_server.py b/tests/test_start_server.py new file mode 100644 index 0000000..f53604b --- /dev/null +++ b/tests/test_start_server.py @@ -0,0 +1,142 @@ +"""The HTTP launchers serve the user's project, never the Daem0n-MCP checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest + +# Import before patch.dict(sys.modules) snapshots it, so the patched launcher is +# the one start_server imports in every test (not a fresh re-import). +import daem0nmcp.api.v7.launcher # noqa: F401 + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _launched_project_root(argv: list[str], environ: dict[str, str]) -> str: + import start_server + + seen: list[str] = [] + + def create_server(transport: str, *, host: str | None = None) -> object: + seen.append(os.environ["DAEM0NMCP_PROJECT_ROOT"]) + return object() + + fake_server = types.ModuleType("daem0nmcp.server") + fake_server.create_server = create_server + with ( + patch.dict(sys.modules, {"daem0nmcp.server": fake_server}), + patch.object(sys, "argv", ["start_server.py", *argv]), + patch.dict(os.environ, environ, clear=True), + patch("daem0nmcp.api.v7.launcher.run_server"), + ): + start_server.main() + return seen[0] + + +def test_exported_project_root_survives(tmp_path): + exported = tmp_path / "exported" + exported.mkdir() + root = _launched_project_root([], {"DAEM0NMCP_PROJECT_ROOT": str(exported)}) + assert root == str(exported.resolve()) + + +def test_project_argument_wins_over_exported_root(tmp_path): + chosen = tmp_path / "chosen" + chosen.mkdir() + (tmp_path / "exported").mkdir() + root = _launched_project_root( + ["--project", str(chosen)], + {"DAEM0NMCP_PROJECT_ROOT": str(tmp_path / "exported")}, + ) + assert root == str(chosen.resolve()) + + +def test_current_directory_is_the_default(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert _launched_project_root([], {}) == str(tmp_path.resolve()) + + +def test_a_missing_project_directory_is_refused(tmp_path): + """A typo must not create and serve an empty workspace.""" + with pytest.raises(SystemExit) as raised: + _launched_project_root(["--project", str(tmp_path / "typo")], {}) + assert raised.value.code == 2 + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +@pytest.mark.parametrize("use_argument", [False, True]) +def test_batch_launcher_passes_the_callers_project(tmp_path, use_argument): + """The .bat changes to the repo dir, so it must capture the project first.""" + recorded, environment = _shim(tmp_path) + project = tmp_path / "my project (x86) ^ !name!" + project.mkdir() + + _run_batch( + ["my project (x86) ^ !name!"] if use_argument else [], + cwd=tmp_path if use_argument else project, + environment=environment, + ) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == f'start_server.py --port 9876 --project "{project}\\."' + assert Path(f"{project}\\.").resolve() == project.resolve() + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_leaves_an_exported_project_root_alone(tmp_path): + recorded, environment = _shim(tmp_path) + exported = tmp_path / "exported" + exported.mkdir() + environment["DAEM0NMCP_PROJECT_ROOT"] = str(exported) + + _run_batch([], cwd=tmp_path, environment=environment) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == "start_server.py --port 9876" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_refuses_the_windows_system_directory(tmp_path): + """ "Run as administrator" starts in System32; that is never the project.""" + recorded, environment = _shim(tmp_path) + system32 = Path(os.environ["SYSTEMROOT"]) / "System32" + + completed = _run_batch([], cwd=system32, environment=environment, check=False) + + assert completed.returncode == 2 + assert not recorded.exists() + + +def _shim(tmp_path: Path) -> tuple[Path, dict[str, str]]: + """Put a python.cmd on PATH that records the arguments it is given.""" + shim = tmp_path / "shim" + shim.mkdir() + recorded = tmp_path / "argv.txt" + (shim / "python.cmd").write_text(f'@echo %*> "{recorded}"\r\n', encoding="ascii") + environment = {**os.environ, "PATH": f"{shim}{os.pathsep}{os.environ['PATH']}"} + environment.pop("DAEM0NMCP_PROJECT_ROOT", None) + return recorded, environment + + +def _run_batch( + arguments: list[str], + *, + cwd: Path, + environment: dict[str, str], + check: bool = True, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["cmd", "/c", str(REPO_ROOT / "start_daem0nmcp_server.bat"), *arguments], + cwd=cwd, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + check=check, + timeout=30, + ) diff --git a/tests/test_vectors.py b/tests/test_vectors.py index ac78d0e..f5420a9 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -62,16 +62,16 @@ def test_hybrid_fallback_to_tfidf(self): class TestEncodeDecode: """Test vector encoding and decoding.""" - def test_encode_reports_models_local_remediation_when_unavailable(self): + def test_encode_reports_models_local_remediation_when_unavailable( + self, monkeypatch + ): """The disabled models-local profile exposes its explicit remediation.""" - if not vectors.is_available(): - from daem0nmcp.capabilities import CapabilityUnavailableError - - with pytest.raises( - CapabilityUnavailableError, match="models-local" - ) as error: - vectors.encode("test text") - assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) + from daem0nmcp.capabilities import CapabilityUnavailableError + + monkeypatch.setenv("DAEM0NMCP_MODELS_LOCAL_ENABLED", "false") + with pytest.raises(CapabilityUnavailableError, match="models-local") as error: + vectors.encode("test text") + assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) def test_decode_empty_returns_none(self): """decode returns None for empty bytes.""" From b003c03791955b21b833563c0e277274f1101090 Mon Sep 17 00:00:00 2001 From: Devil <69057727+9thLevelSoftware@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:55:27 -0400 Subject: [PATCH 67/72] PR 5/12: Exempt user text from the path guard, with one predicate for everything else (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Brownfield run 3403b448, PR 5 of 12** — audit-and-fix stack for the v7 core (#61/#62). Base: `v7/bf-01-make-ci-green-and-pin-workspace`. ## Plan (contract) ### PR 5: Exempt user text from the path guard, with one predicate for everything else - Depends on: PR 1 - Files: daem0nmcp/api/v7/models.py, daem0nmcp/api/v7/tools.py, daem0nmcp/api/v7/pinned.py, daem0nmcp/api/v7/runtime_services.py, daem0nmcp/api/v7/operations.py, daem0nmcp/api/v7/record_operations.py, tests/api_v7/test_path_policy.py - Traces to: FP-2, FP-3, F-005, F-006, F-025, F-008, F-061, F-064, F-071, A-006, A-012, UD-3 - Risk: medium - Description (KD-2): 1. **`UserText` type.** In `models.py`, add `UserText = Annotated[str, …]` and keep one `is_host_absolute_path()`. `WireModel`'s model validator skips fields whose annotation is `UserText` (or `list[UserText]`/`dict[str, UserText]` for tags and context values) and checks every other string, input and output. `RelativePath` keeps its validator. 2. **Annotate the user-text fields** in the input and output models (`tools.py`/`models.py`): - memory content, rationale, tags, context values; - recall queries, trigger patterns, outcome text, preflight description; - rule lists `must_do`/`must_not`/`ask_first` (`tools.py:381-383, 603-605, 1410-1423`) and entity names; - output record summaries, recall excerpts/highlights, `meta.warnings` that quote content, dreaming and community summaries, and code-index symbols. 3. **Delete the drifted guards.** Delete `_path_safe_success` and both `_contains_raw_path` copies (`pinned.py:80-110`, `runtime_services.py:124-140`). Route `operations.py:_reject_raw_paths` through `is_host_absolute_path`. 4. **Visibility** (F-061). Validate `context.visibility` as a `Literal` of the values `retrieval/projections.py` accepts. 5. **Filesystem-sink checklist** (R-23), in the PR description. List every input field that reaches `Path(...)`, `open`, `os.*` or subprocess, and confirm each is `RelativePath` (or an https URL for ingest). 6. **Tests** (`test_path_policy.py`). - Store `\\d+`, `/health`, `GET /api/v1/users` and `C:\proj\src\app.py` content as `decision` and as `warning`: the store returns a `record_id`, and brief, recall, preflight and outcome all succeed. - A pre-fix poisoned record inserted directly reads back fine. - `relative_file_path` values `C:\x` and `/etc/x` are rejected. - Round-trip property: every string accepted in a `UserText` input field validates in every output model field that can carry it. - **Mechanical coverage** (R-26). Enumerate every free-text-typed field (`str`/`MediumText`/`ShortText` aliases, including inside lists and dicts) of every write tool's input model *from the models themselves*, not from a list. Seed each with `C:\proj\x.py`, `/health` and `\d+`: rules, triggers, entities, context and memories. Also insert the same strings directly into `memory_records`, `governance_*` and entity rows, the way a migrated v6 DB carries them. Then call every read tool and resource template (reusing PR 1's harness), and assert that none returns an error. An unannotated user-derived field fails this test. - Model-wide check: a server-generated field (for example a health diagnostic) set to `C:\x` fails validation. - PR 1's sweep, including its leak assertion, stays green. - Acceptance: - The new tests pass. - `_path_safe_success` and `_contains_raw_path` no longer exist. - Exactly one absolute-path regex set remains in `daem0nmcp/api/v7`. - The checklist is in the PR description. --- ## What landed # PR 5 summary: Exempt user text from the path guard, with one predicate for everything else (Saved by the orchestrator from the implementer's reply.) Branch `v7/bf-05-exempt-user-text-from-the-path-g`, base f7f024a (PR 1 tip). Worktree `D:\Daem0n-MCP\.claude\worktrees\agent-ab8e615d18037cdb7`. - 224e7ee `fix: let user text mention paths, with one path predicate for everything else` - e6a087e `test: prove path-bearing user text round-trips through every tool` - 9ef43d1 `test: pin every read against its pre-path baseline` (final) 20 files, +1046/−344. Production: `api/v7/{models,tools,resources,pinned,runtime_services,operations,resource_repository,discovery_operations,code_entity_operations,intelligence_operations,utility_operations}.py`, `capture_candidates.py`, `edit_bridge.py`. Tests: new `tests/api_v7/test_path_policy.py`; updated `test_process_workspace_isolation_sweep.py`, `test_models.py`, `test_operations.py`, `test_pinned_handlers.py`, `test_resource_repository.py`, `test_runtime_services.py`. ## What and why 1. **One predicate (`models.py`)**: `is_host_absolute_path(str)` is the only predicate/regex set (Windows, POSIX, `file:` URI; `_WINDOWS_DRIVE` for RelativePath). `UserText = Annotated[str, _USER_TEXT]` and `UserJsonObject` (ContextJsonObject built on it). `guarded_strings(value)` walks fields by their own annotations, skipping any carrying the marker (bare/Optional/list/set/dict). `WireModel.reject_absolute_filesystem_paths` checks every guarded string on input and output. `contains_absolute_filesystem_path` kept as a thin wrapper (edit-bridge native-argument check, legacy_compat). `discovery_operations._safe_code_path` uses `_relative_path`. 2. **Annotated fields**: tools.py aliases `UserShortText`, `UserNameText`, `UserMediumText`, `UserLongText`, `OptionalUserMediumText`, `RecordContext` (unused ShortText/LongText/OptionalMediumText deleted). Inputs: memory content/rationale/context/tags/procedure steps (store, batch, capture promote); all queries; preflight description and target_arguments; edit description; outcome text; rule trigger/must_do/must_not/ask_first/warnings (create + RulePatch); trigger pattern/recall query; entity names, qualified_name; compress/verify text; sandbox code; link description/label; active-context reason; ingest topic; refactor objective; compaction summary/query; debate topic/positions; brief focus_areas. Outputs: TriggerView pattern/query, OutcomeSummary excerpt, PreflightGuidance/RuleCheckData lists, TriggerMatch.matched_value, community/graph-node/entity labels and names, TextSearchHit excerpt, compress text, DocumentSource topic, verify claims, sandbox stdout/stderr, CodeEntitySummary.qualified_name, TodoFinding.text, refactor proposal, relationship description, mermaid text, EntityEvolutionItem.summary, export event/vector/legacy-row payloads, CaptureCandidateView.proposed_record, link label, debate rounds/synthesis; models.py RecordSummary.excerpt, EvidenceItem.bounded_excerpt, RetrievalData.rendered_context, Tag; resources.py RuleView text fields, ActiveContextItem.reason. **Still guarded**: diagnostics, remediation, covenant reasons, UpdateSummary, differences, rule-evolution summary, contradiction explanation, sanitized_logs, refactor warnings, health, manifests, provenance. 3. **Drifted guards deleted**: pinned.py `_path_safe_success`, `_contains_raw_path` + regexes, guidance internal_error branch; runtime_services.py `_contains_raw_path` + 7 call sites (all on validated WireModels); `operations._reject_raw_paths` no longer scans free text — refuses non-empty `file_path`/`project_path`/`database_path` and absolute `relative_file_path`/`file_path_relative`, not descending into `context`, still refuses bytes (before, any memory mentioning `/health` broke export/versions/at-time); also removed `resource_repository._reject_canonical_root` (a warning naming the root made the resource unreadable) and path checks in `code_entity._safe_event_content`, `intelligence._safe_text`, TODO scan text, capture-candidate record, edit-preflight description (only hash stored). 4. **Visibility (F-061)**: `RecordContext` validates `context.visibility` ∈ {workspace, private, shared} (the set projections accept) on store/batch/promote; before, "team" was committed then INTERNAL_ERROR. 5. **Filesystem-sink checklist**: `code_index.relative_root`, `patterns[]`, `code_todos_scan*.relative_root`, `code_refactor_propose.relative_file_path` → RelativePath; `memory_recall_file`/`context_triggers_match.relative_file_path` → SQL/glob only; `memory_store.relative_file_path` stored never opened; `document_ingest_url.url` HttpsUrl (https, no credentials); `sandbox_execute_python.code` → remote E2B. No v7 input reaches subprocess (only fixed git invocation in resource_repository). 6. **Sweep (PR 1)**: B and C each store a warning naming `/src/neutral.py`. Scan validates each response against its own model (`ApiResponse[TOOL_DATA_MODELS[tool]]` / resource document model, strict=False), then checks only `guarded_strings()` for root/home needles — exemption by annotation only. New `test_leak_check_exempts_user_text_by_annotation_only` proves a server-generated field carrying the root fails. Pinned outcomes unchanged (70 tools, 4 resources, B and C, JWT pass). ## Tests - `test_path_policy.py` (13 tests): model-level rejects `C:\x`, `/etc/x` in relative_file_path; user-text inputs accept 7 strings; guarded diagnostics/remediation with paths fail; >30 UserText output fields accept each string via WireModel probe, guarded MediumText refuses. `test_path_content_is_stored_and_read_back`: pre-fix `\\d+` warning inserted via EventStore appears in session_brief; `\\d+`, `/health`, `GET /api/v1/users`, `C:\proj\src\app.py` stored as decision and warning → brief, recall, text search, preflight, outcome succeed, 4 resources read. `test_every_free_text_field_accepts_paths_and_every_read_survives` (R-26): ~60 free-text fields across 31 tools discovered from model annotations, seeded with `C:\proj\x.py`, `/health`, `\d+`; seeded outcome equals neutral or is a non-refusal; plus direct inserts (records, rule, file trigger, entities, community); every read tool + 4 resources compared to pre-path baseline — none degrades. Mutations (guarding TriggerView.pattern on output or input; guarding EntitySummary.name) fail the test. - `test_path_policy.py` 13 passed (~75 s); sweep 3 passed (37 s); ruff/format clean; mypy api/v7 clean (native, linux); release inventory valid. - Full suite: 2815 passed, 121 skipped, 1 failed (`test_database.py::TestSQLitePragmas::test_foreign_keys_enabled`, WinError 32 on `.migrate-v7.lock` — file-lock flake with other agents on the machine; passes on rerun). Earlier `-x` run hit one `disk I/O error` in test_runtime_services; passes on rerun. CI not yet run. ## Deviations 1. Files beyond plan list (each had a user-text refusal or leftover regex found by the mechanical test): resources.py, resource_repository.py, discovery_operations.py, code_entity_operations.py, intelligence_operations.py, utility_operations.py, capture_candidates.py, edit_bridge.py. record_operations.py unchanged (file_path integrity check is PR 6). 2. PR 1 sweep changed to pass with host paths in user text. 3. Six existing tests rewritten (asserted old refusal of paths in free text). 4. Whole JSON fields exempt incl. keys (ExportEvent.payload, vector payloads, legacy_rows, proposed_record, preflight target_arguments, context); structured path keys in event payloads still refused by `_reject_raw_paths`. 5. `document_ingest_url.topic` and `memory_compact.summary` seeds only reach a pre-content refusal (acceptance checked, not storage). 6. `workspace_import` bundles not seeded. ## Follow-ups (not fixed) - Control characters accepted on input but refused on output by RecordSummary.excerpt and Tag — same stored-then-bricked pattern, not paths. - Model-wide POSIX regex refuses some valid relative paths (`src/__pycache__/x`) and globs (`src/**/foo.py`). - edit_bridge refuses native edit arguments with absolute paths, but Claude Code's Edit sends absolute file_path (PR 2 area). - Migrated absolute `memory_records.file_path` is PR 6 scope. ## Round 2 0a204d3 + 88e66ff + 797685e. 24 addressed, 1 wontfix (non-path migrated-row limits → PR 6). Key: `ErrorRemedy.arguments` → UserJsonObject (token errors no longer crash on path content); control characters read back (`_sanitized` removed from Tag/RecordSummary.excerpt/EvidenceItem.bounded_excerpt and from intelligence/code-entity checks); mechanical coverage test walks nested models from annotations, asserts on unclassified shapes, covers workspace_import via a real export→import round trip, requires non-refused neutral calls, pins GATED_READS and a ≥60 seed floor; vector payloads guarded again, ExportEvent.payload exempt only for `data`, legacy pages through `_reject_raw_paths` with RelativePath validation; leak sweep scans every string allowing root/home only in a marked UserText note; sanitized_logs UserText(4096); RelativePath skips the substring scan (globs OK); legacy_compat keeps path context; `_safe_code_path` deleted; `test_memory.py::test_v7_legacy_id_keeps_opaque_canonical_origin` updated to expect kept context (still asserts opaque ids/evidence). All 8 reviewer mutations now fail. CI run 35439309031 green on all 9 legs + lint/type-check/release-inventory. --- ## Review # PR 5 review — merged (round 2: all addressed; general/tests/plan-alignment verified; CI 35439309031 green) Reviewer slots: general, general-2, tests, security, plan-alignment (effort 5). Item statuses: 24 addressed, 1 wontfix. CI: https://github.com/9thlevelsoftware/Daem0n-MCP/actions/runs/35439309031 (green) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- daem0nmcp/api/v7/code_entity_operations.py | 10 +- daem0nmcp/api/v7/discovery_operations.py | 19 - daem0nmcp/api/v7/intelligence_operations.py | 10 +- daem0nmcp/api/v7/models.py | 131 ++- daem0nmcp/api/v7/operations.py | 40 +- daem0nmcp/api/v7/pinned.py | 53 +- daem0nmcp/api/v7/resource_repository.py | 29 - daem0nmcp/api/v7/resources.py | 6 +- daem0nmcp/api/v7/runtime_services.py | 35 - daem0nmcp/api/v7/tools.py | 231 +++-- daem0nmcp/api/v7/utility_operations.py | 4 +- daem0nmcp/capture_candidates.py | 2 - daem0nmcp/edit_bridge.py | 8 +- daem0nmcp/retrieval/legacy_compat.py | 6 +- tests/api_v7/test_models.py | 45 +- tests/api_v7/test_operations.py | 22 +- tests/api_v7/test_path_policy.py | 921 ++++++++++++++++++ tests/api_v7/test_pinned_handlers.py | 28 +- .../test_process_workspace_isolation_sweep.py | 122 ++- tests/api_v7/test_resource_repository.py | 27 +- tests/api_v7/test_runtime_services.py | 2 +- tests/test_memory.py | 11 +- 22 files changed, 1388 insertions(+), 374 deletions(-) create mode 100644 tests/api_v7/test_path_policy.py diff --git a/daem0nmcp/api/v7/code_entity_operations.py b/daem0nmcp/api/v7/code_entity_operations.py index 5368379..3d45aed 100644 --- a/daem0nmcp/api/v7/code_entity_operations.py +++ b/daem0nmcp/api/v7/code_entity_operations.py @@ -49,7 +49,6 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, ) from .public_ids import ( PublicObjectIdNotFound, @@ -79,7 +78,6 @@ _MAX_ENTITY_EVENTS = 200 _MAX_IMPACT_ENTITIES = 500 _MAX_IMPACT_EDGES = 5_000 -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _EVENT_TYPE_RE = re.compile(r"^[a-z][a-z0-9_.-]{2,79}$") _EVENT_COLUMNS = ( "event_id,workspace_id,stream_id,stream_kind,stream_version,event_type," @@ -935,12 +933,8 @@ def _entity_row( def _safe_event_content(state: Mapping[str, Any]) -> str: content = state.get("content") - if ( - not isinstance(content, str) - or not content - or _CONTROL_RE.search(content) is not None - or contains_absolute_filesystem_path(content) - ): + # Stored user text reads back as stored (UD-3). + if not isinstance(content, str) or not content: raise CodeEntityOperationError("CAPABILITY_DEGRADED") return content diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..009e4d2 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -16,7 +16,6 @@ import inspect import json import os -import posixpath import re import secrets import sqlite3 @@ -755,24 +754,6 @@ def _partition_metadata( return row_count, content_hash -def _safe_code_path(value: object) -> bool: - if ( - not isinstance(value, str) - or not 1 <= len(value) <= 1024 - or "\\" in value - or "\x00" in value - or value.startswith(("/", "~")) - or re.match(r"^[A-Za-z]:", value) is not None - or value in {".", ".."} - ): - return False - components = value.split("/") - return ( - all(component not in {"", ".", ".."} for component in components) - and posixpath.normpath(value) == value - ) - - def _validate_entity_partition( connection: sqlite3.Connection, workspace_id: str, diff --git a/daem0nmcp/api/v7/intelligence_operations.py b/daem0nmcp/api/v7/intelligence_operations.py index c144850..c94d358 100644 --- a/daem0nmcp/api/v7/intelligence_operations.py +++ b/daem0nmcp/api/v7/intelligence_operations.py @@ -34,7 +34,6 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, parse_wire_datetime, ) from .public_ids import PublicObjectIdNotFound, PublicObjectIdRepository @@ -63,7 +62,6 @@ _MAX_COMMUNITY_MEMBERS = 20_000 _CLAIM_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|[\r\n]+") _WORD_RE = re.compile(r"[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*") -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _NEGATIONS = frozenset({"no", "not", "never", "none", "without"}) _STOP_WORDS = frozenset( { @@ -369,12 +367,8 @@ def _verified_event(row: sqlite3.Row) -> dict[str, Any]: def _safe_text(value: object) -> str: - if ( - not isinstance(value, str) - or not value - or _CONTROL_RE.search(value) is not None - or contains_absolute_filesystem_path(value) - ): + # Stored user text reads back as stored (UD-3). + if not isinstance(value, str) or not value: raise IntelligenceOperationError("CAPABILITY_DEGRADED") return value diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index c440e7e..d655b71 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -10,7 +10,9 @@ import json import posixpath import re +from collections.abc import Iterator, Mapping from datetime import datetime, timezone +from functools import cache from typing import ( TYPE_CHECKING, Annotated, @@ -19,6 +21,7 @@ Literal, TypeAlias, TypeVar, + get_args, ) from pydantic import ( @@ -55,26 +58,94 @@ _CONTROL_CHAR = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") -def contains_absolute_filesystem_path(value: object) -> bool: - """Return whether a nested public value contains an absolute path.""" +def is_host_absolute_path(value: str) -> bool: + """Return whether text contains an absolute host path or file URI. + + This is the only absolute-path predicate on the v7 wire. ``WireModel`` + applies it to every string except ``UserText`` fields. + """ + + return bool( + _WINDOWS_ABSOLUTE_PATH.search(value) + or _POSIX_ABSOLUTE_PATH.search(value) + or _FILE_URI.search(value) + ) + + +class _Exempt: + """``Annotated`` marker for strings the substring path scan skips.""" + + __slots__ = ("name",) + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return self.name + + +_USER_TEXT = _Exempt("UserText") +# RelativePath's own validator refuses absolute and escaping paths exactly; +# the prose scan would also refuse valid names such as ``src/__pycache__/x``. +_RELATIVE_PATH = _Exempt("RelativePath") + +# Memory content, queries, rule text and server text quoting them may mention +# routes and file paths ("/health", "C:\proj\app.py"). Fields typed with it, +# alone or inside a list, set or dict, are exempt from the absolute-path rule in +# both directions, so anything accepted on input can always be read back. +UserText = Annotated[str, _USER_TEXT] + + +def _carries(annotation: object, marker: _Exempt) -> bool: + return annotation is marker or any( + _carries(argument, marker) for argument in get_args(annotation) + ) + + +def _fields_marked(model: type[BaseModel], marker: _Exempt) -> frozenset[str]: + return frozenset( + name + for name, field in model.model_fields.items() + if marker in field.metadata or _carries(field.annotation, marker) + ) + + +@cache +def _user_text_fields(model: type[BaseModel]) -> frozenset[str]: + return _fields_marked(model, _USER_TEXT) + + +@cache +def _exempt_fields(model: type[BaseModel]) -> frozenset[str]: + return _user_text_fields(model) | _fields_marked(model, _RELATIVE_PATH) + + +def guarded_strings(value: object, user_text: bool = False) -> Iterator[str]: + """Yield every string, keys included, that the absolute-path rule covers. + + Nested models are walked with their own field annotations, so a model + inside a ``UserText`` container is still checked. + """ - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) - or _POSIX_ABSOLUTE_PATH.search(value) - or _FILE_URI.search(value) - ) - if isinstance(value, dict): - return any( - contains_absolute_filesystem_path(key) - or contains_absolute_filesystem_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(contains_absolute_filesystem_path(item) for item in value) if isinstance(value, BaseModel): - return contains_absolute_filesystem_path(value.__dict__) - return False + exempt = _exempt_fields(type(value)) + for name, item in value.__dict__.items(): + yield from guarded_strings(item, name in exempt) + elif isinstance(value, Mapping): + for key, item in value.items(): + yield from guarded_strings(key, user_text) + yield from guarded_strings(item, user_text) + elif isinstance(value, (list, tuple, set, frozenset)): + for item in value: + yield from guarded_strings(item, user_text) + elif isinstance(value, str) and not user_text: + yield value + + +def contains_absolute_filesystem_path(value: object) -> bool: + """Return whether a nested value has a guarded string with a host path.""" + + return any(is_host_absolute_path(text) for text in guarded_strings(value)) def _relative_path(value: str) -> str: @@ -283,6 +354,7 @@ def _document_validation_error( str, StringConstraints(strict=True, min_length=1, max_length=1024), AfterValidator(_relative_path), + _RELATIVE_PATH, ] AwareDateTime = Annotated[datetime, BeforeValidator(parse_wire_datetime)] UtcDateTime = Annotated[AwareDateTime, AfterValidator(_utc)] @@ -326,7 +398,8 @@ def _document_validation_error( Field(max_length=MAX_JSON_COLLECTION_ITEMS), ], ) -ContextJsonObject = Annotated[JsonObject, AfterValidator(_context_size)] +UserJsonObject = Annotated[JsonObject, _USER_TEXT] +ContextJsonObject = Annotated[UserJsonObject, AfterValidator(_context_size)] ErrorCodeValue = Annotated[ErrorCode, BeforeValidator(_error_code)] UpperSnakeCode = Annotated[ @@ -348,9 +421,8 @@ def _document_validation_error( ), ] Tag = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=80), - AfterValidator(_sanitized), ] ProviderName = Annotated[ str, @@ -430,7 +502,7 @@ class WireModel(BaseModel): @model_validator(mode="after") def reject_absolute_filesystem_paths(self) -> WireModel: - if contains_absolute_filesystem_path(self.__dict__): + if contains_absolute_filesystem_path(self): raise ValueError("absolute filesystem paths are forbidden on the v7 wire") return self @@ -471,7 +543,8 @@ class FieldError(WireModel): class ErrorRemedy(WireModel): tool: ToolName - arguments: JsonObject = Field(default_factory=dict) + # The caller's own validated arguments, echoed back for the retry. + arguments: UserJsonObject = Field(default_factory=dict) Remedy = ErrorRemedy @@ -586,9 +659,8 @@ class RecordSummary(WireModel): record_id: RecordId record_type: RecordType excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), - AfterValidator(_sanitized), ] tags: Annotated[list[Tag], AfterValidator(_unique_strings)] = Field( default_factory=list, @@ -621,9 +693,8 @@ class EvidenceItem(WireModel): citation: Citation record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), - AfterValidator(_sanitized), ] channels: Annotated[list[ProviderName], AfterValidator(_unique_strings)] = Field( min_length=1, @@ -694,7 +765,7 @@ class RetrievalData(WireModel): items: list[EvidenceItem] = Field(default_factory=list, max_length=50) rendered_context: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] | None @@ -790,6 +861,8 @@ class Preview(WireModel): "EventId", "FactId", "FieldError", + "guarded_strings", + "is_host_absolute_path", "JsonObject", "JsonValue", "MAX_CONTEXT_JSON_BYTES", @@ -822,6 +895,8 @@ class Preview(WireModel): "TokenUsage", "ToolName", "TriggerId", + "UserJsonObject", + "UserText", "UtcDateTime", "VersionId", "WireModel", diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..ff1e6b4 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -24,6 +24,8 @@ from types import MappingProxyType from typing import Any +from pydantic import TypeAdapter, ValidationError + from ...bounded_workers import BoundedWorkerBusyError, BoundedWorkerPool from ...covenant import CovenantGate, InvocationScope from ...event_store import ( @@ -51,7 +53,7 @@ from ...workspace import Workspace from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import EvidenceRef, Page, RecordSummary +from .models import EvidenceRef, Page, RecordSummary, RelativePath from .portable_projections import ( ImportFinalizationLease, PortableTransferError, @@ -89,8 +91,10 @@ _MAX_EXPORT_EVENTS = 10_000 _IMPORT_LEASE_RENEW_MARGIN_US = 5 * 60 * 1_000_000 _CURSOR_RE = re.compile(r"^cur_([0-9a-f]{64})$") -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? None: def _reject_raw_paths(value: object) -> None: - if isinstance(value, str): - if ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ): - raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - elif isinstance(value, Mapping): + """Refuse host paths in the structured path fields of an event payload. + + Free text (content, rationale, rule text, context) is user text and may + mention paths, as on the wire. + """ + if isinstance(value, Mapping): for key, item in value.items(): - if ( - key in {"file_path", "project_path", "database_path"} - and item is not None - and item != "" - ): + if key in _RAW_PATH_KEYS and item is not None and item != "": raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - _reject_raw_paths(item) + if key in _RELATIVE_PATH_KEYS and item is not None: + try: + _RELATIVE_PATH_ADAPTER.validate_python(item) + except ValidationError: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") from None + if key != "context": + _reject_raw_paths(item) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): for item in value: _reject_raw_paths(item) @@ -993,6 +998,8 @@ def _export_sync( content = page["content"] manifest = page["manifest"] items = content["items"] + if page["page_kind"] == "legacy": + _reject_raw_paths(items) descriptor = page.get( "page_descriptor", { @@ -1051,6 +1058,7 @@ def _portable_page(bundle: ExportBundle) -> dict[str, Any]: items = [event.model_dump(mode="json") for event in bundle.events] elif bundle.page_kind == "legacy": items = list(bundle.legacy_rows) + _reject_raw_paths(items) else: items = [point.model_dump(mode="json") for point in bundle.vector_points] return { diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 9b612ba..1a873ef 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -4,12 +4,11 @@ import inspect import os -import re from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType -from typing import Any, Protocol, TypeVar +from typing import Any, Protocol from ...covenant import ( ArgumentNormalizationError, @@ -78,38 +77,6 @@ def __init__(self, code: str) -> None: super().__init__(code) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - - -T = TypeVar("T") - - -def _path_safe_success(response: ResponseContext, data: T) -> ApiResponse[T]: - if _contains_raw_path(data): - return response.internal_error() - return response.success(data) - - _EXPECTED_SERVICE_ERRORS = MappingProxyType( { "NOT_FOUND": ( @@ -437,7 +404,7 @@ async def session_brief( data = SessionBriefData.model_validate(assembled) if data.workspace_id != request.workspace_id: raise ValueError("briefing service returned a mismatched workspace") - result = _path_safe_success(response, data) + result = response.success(data) if result.ok: self._dependencies.covenant_gate.record_briefing(scope) return result @@ -538,11 +505,8 @@ async def memory_preflight( else guidance_value ) guidance = PreflightGuidance.model_validate(guidance_result) - if _contains_raw_path(guidance): - return response.internal_error() if not target_is_complete: - return _path_safe_success( - response, + return response.success( PreflightData( guidance=guidance, preflight_token=None, @@ -563,7 +527,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=expires_at, ) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc @@ -744,8 +708,7 @@ async def authorize_linked(selected: Workspace) -> None: if inspect.isawaitable(result_value) else result_value ) - return _path_safe_success( - response, + return response.success( RetrievalData.model_validate(result), ) except Exception as exc: @@ -866,7 +829,7 @@ async def memory_store( stream_version=stored.event.stream_version, idempotent_replay=stored.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -961,7 +924,7 @@ async def memory_record_outcome( worked=recorded.worked, idempotent_replay=recorded.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -1029,7 +992,7 @@ async def system_health( return failure if not request.include_components and data.capability_states: data = data.model_copy(update={"capability_states": []}) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index c1c7a07..30f5ebc 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -336,32 +336,6 @@ def _string_list(value: object) -> list[str]: return decoded -def _workspace_root_tokens(workspace: Workspace) -> tuple[str, ...]: - root = str(workspace.root) - variants = {root.casefold(), root.replace("\\", "/").casefold()} - return tuple(value for value in variants if value) - - -def _reject_canonical_root(value: object, workspace: Workspace) -> None: - tokens = _workspace_root_tokens(workspace) - - def walk(item: object) -> None: - if isinstance(item, str): - normalized = item.casefold() - portable = item.replace("\\", "/").casefold() - if any(token in normalized or token in portable for token in tokens): - raise ValueError("public value contains a canonical workspace root") - elif isinstance(item, Mapping): - for key, child in item.items(): - walk(key) - walk(child) - elif isinstance(item, (list, tuple)): - for child in item: - walk(child) - - walk(value) - - def _validated_now(clock: Callable[[], datetime]) -> datetime: value = clock() if ( @@ -1145,7 +1119,6 @@ def _record_summary( "created_at": _datetime_from_us(row["created_at_us"]), "updated_at": _datetime_from_us(row["updated_at_us"]), } - _reject_canonical_root(public_value, workspace) return RecordSummary.model_validate(public_value), deleted def _read_rules_sync( @@ -1191,7 +1164,6 @@ def _read_rules_sync( "enabled": _flag(row["enabled"]), "created_at": _datetime_from_us(row["created_at_us"]), } - _reject_canonical_root(public_value, workspace) values.append(RuleView.model_validate(public_value)) return values finally: @@ -1417,7 +1389,6 @@ def _read_active_context_sync( "added_at": added_at, "expires_at": expires_at, } - _reject_canonical_root(public_value, workspace) item = ActiveContextItem.model_validate(public_value) ordered_values.append( ( diff --git a/daem0nmcp/api/v7/resources.py b/daem0nmcp/api/v7/resources.py index e19f4c5..b488586 100644 --- a/daem0nmcp/api/v7/resources.py +++ b/daem0nmcp/api/v7/resources.py @@ -23,6 +23,7 @@ AwareDateTime, RecordSummary, RuleId, + UserText, UtcDateTime, WireModel, WorkspaceId, @@ -49,8 +50,9 @@ _UTC_DATETIME_ADAPTER = TypeAdapter(UtcDateTime) _ACCESS_FAILURE = object() +# Rule text is user-authored and may mention paths. BoundedText = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] @@ -91,7 +93,7 @@ class ActiveContextItem(WireModel): priority: Annotated[int, Field(strict=True, ge=-100, le=100)] reason: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index aada0f8..09d08c3 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -11,7 +11,6 @@ import inspect import json import os -import re import sqlite3 import threading import time @@ -89,8 +88,6 @@ SessionBriefInput, ) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) or _POSIX_ABSOLUTE_PATH.search(value) - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - - def _validated_workspace(workspace: Workspace) -> Workspace: if not isinstance(workspace, Workspace): raise RuntimeServiceError("INVALID_WORKSPACE") @@ -1134,8 +1113,6 @@ async def retrieve( raise except Exception: raise RuntimeServiceError("RETRIEVAL_FAILED") from None - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return hydrated async def _retrieve_source( @@ -1254,8 +1231,6 @@ async def retrieve_one( if inspect.isawaitable(value): await value hydrated = compose_federated_results(dict(pairs), query) - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") finally: guard.release() except asyncio.TimeoutError: @@ -1613,8 +1588,6 @@ async def assemble( value = self._reader(workspace, request) if inspect.isawaitable(value): value = await value - if _contains_raw_path(value): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") data = SessionBriefData.model_validate(value) except asyncio.CancelledError: raise @@ -1624,8 +1597,6 @@ async def assemble( raise RuntimeServiceError("BRIEFING_FAILED") from None if data.workspace_id != workspace.workspace_id: raise RuntimeServiceError("BRIEFING_FAILED") - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data @@ -1680,8 +1651,6 @@ async def guidance( raise except Exception: raise RuntimeServiceError("PREFLIGHT_FAILED") from None - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data @@ -1824,8 +1793,6 @@ def __init__( ) except Exception: raise ValueError("health service configuration is invalid") from None - if _contains_raw_path(self._data): - raise ValueError("health service configuration is unsafe") self._workers = BoundedWorkerPool( max_workers=2, thread_name_prefix="daem0nmcp-v7-health", @@ -1904,8 +1871,6 @@ async def inspect( "runtime_diagnostics": diagnostics, } ) - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data diff --git a/daem0nmcp/api/v7/tools.py b/daem0nmcp/api/v7/tools.py index fb5d7bc..b1fa7c9 100644 --- a/daem0nmcp/api/v7/tools.py +++ b/daem0nmcp/api/v7/tools.py @@ -53,19 +53,18 @@ Tag, ToolName, TriggerId, + UserJsonObject, + UserText, UtcDateTime, VersionId, WireModel, WorkspaceId, + contains_absolute_filesystem_path, ) from .policy import V7_TOOL_LEVELS, V7ArgumentNormalizer from .registry import PINNED_TOOL_NAMES, ManifestError, ToolSpec from .resources import ActiveContextItem, RuleView -ShortText = Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=80), -] NameText = Annotated[ str, StringConstraints(strict=True, min_length=1, max_length=256), @@ -74,17 +73,34 @@ str, StringConstraints(strict=True, min_length=1, max_length=2000), ] -LongText = Annotated[ - str, +# User-authored or user-derived text, which may mention paths (see UserText). +UserShortText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=80), +] +UserNameText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=256), +] +UserMediumText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=2000), +] +UserLongText = Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] -OptionalMediumText = ( - Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=2000), - ] - | None -) +OptionalUserMediumText = UserMediumText | None +RecordVisibility = Literal["workspace", "private", "shared"] + + +def _record_visibility(context: dict[str, object]) -> dict[str, object]: + if context.get("visibility", "workspace") not in get_args(RecordVisibility): + raise ValueError("context.visibility must be workspace, private or shared") + return context + + +RecordContext = Annotated[ContextJsonObject, AfterValidator(_record_visibility)] IdempotencyKey = Annotated[ str, StringConstraints( @@ -330,8 +346,8 @@ class DiagnosticSummary(WireModel): class TriggerView(WireModel): trigger_id: TriggerId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool updated_at: UtcDateTime @@ -346,7 +362,7 @@ class OutcomeSummary(WireModel): record_id: RecordId worked: bool outcome_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), ] happened_at: UtcDateTime @@ -378,10 +394,10 @@ class SessionBriefData(WireModel): class PreflightGuidance(WireModel): records: list[RecordSummary] = Field(default_factory=list, max_length=20) rules: list[RuleView] = Field(default_factory=list, max_length=20) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class PreflightData(WireModel): @@ -413,7 +429,7 @@ class EditPreflightData(WireModel): class CaptureCandidateView(WireModel): candidate_id: CaptureCandidateId source_kind: Literal["native_edit", "tool_result", "dreaming", "system"] - proposed_record: JsonObject + proposed_record: UserJsonObject provenance: JsonObject created_at: UtcDateTime @@ -521,7 +537,7 @@ class HealthData(WireModel): class TriggerMatch(WireModel): trigger: TriggerView - matched_value: NameText + matched_value: UserNameText records: list[RecordSummary] = Field(default_factory=list, max_length=20) @@ -564,7 +580,7 @@ class HierarchyLayer(WireModel): class CommunitySummary(WireModel): community_id: CommunityId - label: NameText + label: UserNameText level: Annotated[int, Field(ge=0, le=32)] member_count: Annotated[int, Field(ge=0, le=1_000_000)] parent_community_id: CommunityId | None = None @@ -591,7 +607,7 @@ def validate_span(self) -> HighlightSpan: class TextSearchHit(WireModel): record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), ] highlights: list[HighlightSpan] = Field(default_factory=list, max_length=100) @@ -600,15 +616,15 @@ class TextSearchHit(WireModel): class RuleCheckData(WireModel): matched_rules: list[RuleView] = Field(default_factory=list, max_length=50) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class ContextCompressData(WireModel): text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] original_tokens: Annotated[int, Field(ge=0, le=1_000_000)] @@ -625,7 +641,7 @@ class MemoryStoreBatchData(WireModel): class DocumentSource(WireModel): url: HttpsUrl - topic: MediumText + topic: UserMediumText content_hash: ContentHash @@ -637,13 +653,13 @@ class DocumentIngestData(WireModel): class VerifiedClaim(WireModel): - claim: MediumText + claim: UserMediumText status: Literal["supported", "contradicted", "unknown"] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=32) class Contradiction(WireModel): - claim: MediumText + claim: UserMediumText explanation: MediumText evidence_refs: list[EvidenceRef] = Field(min_length=1, max_length=32) @@ -657,11 +673,16 @@ class MemoryVerifyData(WireModel): class SandboxExecutionData(WireModel): success: bool - stdout: Annotated[str, StringConstraints(strict=True, max_length=100_000)] - stderr: Annotated[str, StringConstraints(strict=True, max_length=100_000)] + stdout: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] + stderr: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] exit_status: Annotated[int, Field(ge=-1, le=255)] execution_time_ms: Annotated[int, Field(ge=0, le=60_000)] - sanitized_logs: list[MediumText] = Field(default_factory=list, max_length=100) + # The same lines as stdout/stderr (see external_operations), so user text. + sanitized_logs: list[ + Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=4096) + ] + ] = Field(default_factory=list, max_length=100) class CodeIndexData(WireModel): @@ -675,7 +696,7 @@ class CodeIndexData(WireModel): class CodeEntitySummary(WireModel): code_entity_id: CodeEntityId kind: CodeEntityKind - qualified_name: NameText + qualified_name: UserNameText relative_file_path: RelativePath start_line: Annotated[int, Field(ge=1)] end_line: Annotated[int, Field(ge=1)] @@ -703,7 +724,7 @@ class TodoFinding(WireModel): relative_file_path: RelativePath line: Annotated[int, Field(ge=1)] todo_type: TodoType - text: MediumText + text: UserMediumText class CodeTodosStoreData(WireModel): @@ -714,7 +735,7 @@ class CodeTodosStoreData(WireModel): class RefactorProposalData(WireModel): proposal: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] affected_entities: list[CodeEntitySummary] = Field( @@ -729,7 +750,7 @@ class RelationshipView(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] @@ -752,7 +773,7 @@ class MemoryChainTraceData(WireModel): class KnowledgeGraphNode(WireModel): record: RecordSummary - label: NameText + label: UserNameText node_type: Literal["record", "entity", "community"] @@ -769,7 +790,7 @@ class KnowledgeGraphData(WireModel): class KnowledgeGraphRenderData(WireModel): format: Literal["mermaid"] = "mermaid" text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=200) @@ -795,7 +816,7 @@ class CommunityRebuildData(WireModel): class EntitySummary(WireModel): entity_id: EntityId - name: NameText + name: UserNameText entity_type: EntityType mention_count: Annotated[int, Field(ge=0)] manifest_generation: Annotated[int, Field(ge=1)] @@ -810,7 +831,7 @@ class EntityBackfillData(WireModel): class EntityEvolutionItem(WireModel): happened_at: UtcDateTime - summary: MediumText + summary: UserMediumText event_id: EventId @@ -854,7 +875,15 @@ class ExportEvent(WireModel): event_type: NameText happened_at: UtcDateTime content_hash: ContentHash - payload: JsonObject + # Only the envelope's ``data`` holds user text; the rest stays guarded. + payload: UserJsonObject + + @model_validator(mode="after") + def guard_envelope(self) -> ExportEvent: + envelope = {k: v for k, v in self.payload.items() if k != "data"} + if contains_absolute_filesystem_path(envelope): + raise ValueError("absolute filesystem paths are forbidden on the v7 wire") + return self TransferSessionId = Annotated[ @@ -898,7 +927,7 @@ class ExportBundle(WireModel): page_proof: list[ContentHash] = Field(default_factory=list, max_length=32) next_cursor: Cursor | None = None complete: bool = True - legacy_rows: list[JsonObject] = Field(default_factory=list, max_length=4096) + legacy_rows: list[UserJsonObject] = Field(default_factory=list, max_length=4096) vector_points: list[PortableVectorPoint] = Field( default_factory=list, max_length=4096 ) @@ -970,7 +999,7 @@ class WorkspaceLinkView(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship - label: OptionalMediumText = None + label: OptionalUserMediumText = None class WorkspaceConsolidateData(WireModel): @@ -1016,14 +1045,14 @@ class RuleEvolutionData(WireModel): class DebateRound(WireModel): round_number: Annotated[int, Field(ge=1, le=20)] - advocate: MediumText - challenger: MediumText + advocate: UserMediumText + challenger: UserMediumText class DecisionDebateData(WireModel): rounds: list[DebateRound] = Field(min_length=1, max_length=20) synthesis: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] consensus_record_id: RecordId @@ -1033,7 +1062,7 @@ class DecisionDebateData(WireModel): class SessionBriefInput(WireModel): workspace_id: WorkspaceId - focus_areas: list[ShortText] = Field(default_factory=list, max_length=10) + focus_areas: list[UserShortText] = Field(default_factory=list, max_length=10) warning_limit: Annotated[int, Field(ge=0, le=50)] = 10 failure_limit: Annotated[int, Field(ge=0, le=50)] = 10 @@ -1041,10 +1070,10 @@ class SessionBriefInput(WireModel): class MemoryPreflightInput(WireModel): workspace_id: WorkspaceId target_tool: ProtectedToolName - target_arguments: JsonObject + target_arguments: UserJsonObject description: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=1000), ] | None @@ -1054,7 +1083,7 @@ class MemoryPreflightInput(WireModel): class MemoryRecallInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] limit: Annotated[int, Field(ge=1, le=50)] = 10 @@ -1080,19 +1109,19 @@ def validate_candidate_limit(self) -> MemoryRecallInput: class MemoryStoreInput(WireModel): workspace_id: WorkspaceId record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1106,7 +1135,7 @@ def validate_procedure_steps(self) -> MemoryStoreInput: class EditPreflightInput(WireModel): workspace_id: WorkspaceId edit_request_id: EditRequestId - description: MediumText + description: UserMediumText class MemoryCaptureListInput(WireModel): @@ -1120,11 +1149,11 @@ class MemoryCapturePromoteInput(WireModel): candidate_id: CaptureCandidateId record_type: RecordType content: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8_000), ] - rationale: OptionalMediumText = None - context: ContextJsonObject = Field(default_factory=dict) + rationale: OptionalUserMediumText = None + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=16) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1134,7 +1163,7 @@ class MemoryRecordOutcomeInput(WireModel): workspace_id: WorkspaceId record_id: RecordId outcome_text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] worked: bool @@ -1157,7 +1186,7 @@ class ContextTriggersMatchInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath | None = None tags: list[Tag] = Field(default_factory=list, max_length=32) - entities: list[NameText] = Field(default_factory=list, max_length=32) + entities: list[UserNameText] = Field(default_factory=list, max_length=32) limit: Annotated[int, Field(ge=1, le=20)] = 5 @@ -1182,7 +1211,7 @@ class MemoryRecallFileInput(WireModel): class MemoryRecallEntityInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None cursor: Cursor | None = None limit: Annotated[int, Field(ge=1, le=100)] = 20 @@ -1197,7 +1226,7 @@ def validate_selector(self) -> MemoryRecallEntityInput: class MemoryRecallHierarchicalInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] include_members: bool = False @@ -1207,7 +1236,7 @@ class MemoryRecallHierarchicalInput(WireModel): class MemorySearchTextInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1218,36 +1247,36 @@ class MemorySearchTextInput(WireModel): class RuleCheckInput(WireModel): workspace_id: WorkspaceId - proposed_action: MediumText + proposed_action: UserMediumText context: ContextJsonObject = Field(default_factory=dict) class ContextCompressInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] rate: Annotated[float, Field(ge=0.1, le=1, allow_inf_nan=False)] | None = None - content_type: ShortText | None = None + content_type: UserShortText | None = None preserve_code: bool = True class MemoryCreate(WireModel): record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) + context: RecordContext = Field(default_factory=dict) tags: list[Tag] = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) @model_validator(mode="after") def validate_procedure_steps(self) -> MemoryCreate: @@ -1268,7 +1297,7 @@ class MemoryLinkInput(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] = 1.0 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1296,7 +1325,7 @@ class MemoryPinSetInput(WireModel): class ActiveContextAddInput(WireModel): workspace_id: WorkspaceId record_id: RecordId - reason: OptionalMediumText = None + reason: OptionalUserMediumText = None priority: Annotated[int, Field(ge=-100, le=100)] = 0 expires_at: AwareDateTime | None = None preflight_token: PreflightToken @@ -1317,7 +1346,7 @@ class ActiveContextClearInput(WireModel): class DocumentIngestUrlInput(WireModel): workspace_id: WorkspaceId url: HttpsUrl - topic: MediumText + topic: UserMediumText chunk_size: Annotated[int, Field(ge=256, le=16_000)] = 2000 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1326,7 +1355,7 @@ class DocumentIngestUrlInput(WireModel): class MemoryVerifyInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] categories: RecordTypeSet | None = None @@ -1337,7 +1366,7 @@ class MemoryVerifyInput(WireModel): class SandboxExecutePythonInput(WireModel): workspace_id: WorkspaceId code: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] timeout_seconds: Annotated[int, Field(ge=1, le=60)] = 30 @@ -1356,7 +1385,7 @@ class CodeIndexInput(WireModel): class CodeSearchInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1367,7 +1396,7 @@ class CodeSearchInput(WireModel): class CodeImpactAnalyzeInput(WireModel): workspace_id: WorkspaceId code_entity_id: CodeEntityId | None = None - qualified_name: NameText | None = None + qualified_name: UserNameText | None = None max_depth: Annotated[int, Field(ge=1, le=10)] = 3 @model_validator(mode="after") @@ -1401,27 +1430,27 @@ class CodeTodosScanAndStoreInput(WireModel): class CodeRefactorProposeInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath - objective: OptionalMediumText = None + objective: OptionalUserMediumText = None class RuleCreateInput(WireModel): workspace_id: WorkspaceId - trigger: MediumText - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + trigger: UserMediumText + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] = 0 idempotency_key: IdempotencyKey preflight_token: PreflightToken class RulePatch(WireModel): - trigger: MediumText | None = None - must_do: list[MediumText] | None = Field(default=None, max_length=50) - must_not: list[MediumText] | None = Field(default=None, max_length=50) - ask_first: list[MediumText] | None = Field(default=None, max_length=50) - warnings: list[MediumText] | None = Field(default=None, max_length=50) + trigger: UserMediumText | None = None + must_do: list[UserMediumText] | None = Field(default=None, max_length=50) + must_not: list[UserMediumText] | None = Field(default=None, max_length=50) + ask_first: list[UserMediumText] | None = Field(default=None, max_length=50) + warnings: list[UserMediumText] | None = Field(default=None, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] | None = None enabled: bool | None = None @@ -1449,8 +1478,8 @@ class RuleListInput(WireModel): class ContextTriggerCreateInput(WireModel): workspace_id: WorkspaceId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool = True idempotency_key: IdempotencyKey @@ -1490,7 +1519,7 @@ class KnowledgeGraphGetInput(WireModel): record_ids: RecordIdSet500 | None = None query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1548,7 +1577,7 @@ class EntityBackfillInput(WireModel): class EntityEvolutionTraceInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None include_invalidated: bool = False @@ -1606,13 +1635,13 @@ class MemoryDuplicatesCleanupInput(MemoryDuplicatesPreviewInput): class MemoryCompactionPreviewInput(WireModel): workspace_id: WorkspaceId summary: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] limit: Annotated[int, Field(ge=1, le=100)] = 10 query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1674,7 +1703,7 @@ class WorkspaceLinkInput(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship = "related" - label: OptionalMediumText = None + label: OptionalUserMediumText = None preflight_token: PreflightToken @model_validator(mode="after") @@ -1740,13 +1769,13 @@ class RuleEvolutionAnalyzeInput(WireModel): class DecisionDebateInput(WireModel): workspace_id: WorkspaceId - topic: MediumText + topic: UserMediumText advocate_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] challenger_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] max_rounds: Annotated[int, Field(ge=1, le=20)] = 5 diff --git a/daem0nmcp/api/v7/utility_operations.py b/daem0nmcp/api/v7/utility_operations.py index db5deb2..2d07335 100644 --- a/daem0nmcp/api/v7/utility_operations.py +++ b/daem0nmcp/api/v7/utility_operations.py @@ -22,7 +22,7 @@ from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import Page, contains_absolute_filesystem_path +from .models import Page from .runtime_protocols import WorkerPool from .tasks import await_task_terminal from .tools import ContextCompressData, RefactorProposalData, TodoFinding @@ -278,7 +278,7 @@ def _findings( if todo_type not in selected_types: continue rendered = line.strip()[:2000] - if not rendered or contains_absolute_filesystem_path(rendered): + if not rendered: continue try: findings.append( diff --git a/daem0nmcp/capture_candidates.py b/daem0nmcp/capture_candidates.py index 6856e17..9c56d43 100644 --- a/daem0nmcp/capture_candidates.py +++ b/daem0nmcp/capture_candidates.py @@ -151,8 +151,6 @@ def _validated_record(value: Mapping[str, Any]) -> dict[str, Any]: raise ValueError("capture tags are invalid") if len(tags) != len(set(tags)): raise ValueError("capture tags must be unique") - if contains_absolute_filesystem_path(record): - raise ValueError("capture record contains a public absolute path") return { "record_type": record["record_type"], "content": content, diff --git a/daem0nmcp/edit_bridge.py b/daem0nmcp/edit_bridge.py index 4c02fdd..79f1445 100644 --- a/daem0nmcp/edit_bridge.py +++ b/daem0nmcp/edit_bridge.py @@ -558,11 +558,9 @@ def _issue_receipt_sync( description: str, ) -> EditReceipt: now_us = _datetime_us(self.clock()) - if ( - not isinstance(description, str) - or not 1 <= len(description) <= 2_000 - or contains_absolute_filesystem_path(description) - ): + # The description is user text (it may mention paths) and only its + # hash is stored. + if not isinstance(description, str) or not 1 <= len(description) <= 2_000: raise EditBridgeError("INVALID_ARGUMENT", "edit description rejected") if os.path.normcase(str(workspace.root.resolve())) != scope.canonical_workspace: raise EditBridgeError("TOKEN_SCOPE_MISMATCH", "workspace scope changed") diff --git a/daem0nmcp/retrieval/legacy_compat.py b/daem0nmcp/retrieval/legacy_compat.py index dc405c6..4a80ac9 100644 --- a/daem0nmcp/retrieval/legacy_compat.py +++ b/daem0nmcp/retrieval/legacy_compat.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Any -from ..api.v7.models import contains_absolute_filesystem_path from ..event_store import ( event_hash_for, event_id_for_hash, @@ -57,9 +56,8 @@ def _safe_legacy_context(value: object) -> dict[str, Any] | None: ).encode("utf-8") except (TypeError, ValueError, RecursionError): return None - if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES or contains_absolute_filesystem_path( - value - ): + # Context is user text and may mention paths (UD-3). + if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES: return None # Detach the value from the parsed event payload before it reaches a cache. copied = json.loads(encoded.decode("utf-8")) diff --git a/tests/api_v7/test_models.py b/tests/api_v7/test_models.py index 17b4a9d..511e0fa 100644 --- a/tests/api_v7/test_models.py +++ b/tests/api_v7/test_models.py @@ -247,33 +247,36 @@ def test_json_values_are_strict_finite_bounded_and_duplicate_safe(self) -> None: '{"tool":"memory_store","arguments":{"x":NaN}}' ) - def test_every_wire_model_rejects_absolute_paths_in_free_text_and_json( - self, - ) -> None: + def test_wire_models_reject_absolute_paths_outside_user_text(self) -> None: _, models = _load(self) + # A remedy echoes the caller's own arguments, which may mention paths. + models.ErrorRemedy( + tool="memory_store", + arguments={"context": {"note": "Inspect C:\\private\\secret.txt"}}, + ) with self.assertRaises(ValidationError): - models.ErrorRemedy( - tool="memory_store", - arguments={ - "context": { - "note": "Inspect C:\\private\\secret.txt before release" - } - }, + models.CapabilityState( + name="storage", + status="failed", + remediation="Repair C:\\private\\storage", ) + summary = { + "record_id": RECORD_ID, + "record_type": "warning", + "excerpt": "Do not read /home/private/.ssh/id_rsa", + "tags": ["C:\\proj"], + "relative_file_path": None, + "current_status": "current", + "content_hash": "a" * 64, + "created_at": NOW, + "updated_at": NOW, + } + # Excerpts and tags are user text (UD-3); the path field is not. + models.RecordSummary(**summary) with self.assertRaises(ValidationError): - models.RecordSummary( - record_id=RECORD_ID, - record_type="warning", - excerpt="Do not read /home/private/.ssh/id_rsa", - tags=[], - relative_file_path=None, - current_status="current", - content_hash="a" * 64, - created_at=NOW, - updated_at=NOW, - ) + models.RecordSummary(**{**summary, "relative_file_path": "/home/x"}) class EnvelopeTests(unittest.TestCase): diff --git a/tests/api_v7/test_operations.py b/tests/api_v7/test_operations.py index b042914..becdf49 100644 --- a/tests/api_v7/test_operations.py +++ b/tests/api_v7/test_operations.py @@ -755,23 +755,25 @@ def active_connection(*_args, **_kwargs): self.assertEqual(10_001, bundle.manifest["event_count"]) snapshot_builder.assert_called_once() - async def test_export_rejects_nested_absolute_path_strings(self) -> None: - secret = str(self.fixture.root / "private" / "nested-secret.txt") + async def test_export_carries_user_text_that_mentions_paths(self) -> None: + # Context is user text (UD-3); only the raw path columns are refused. + note = "Crash fixed in C:\\proj\\src\\app.py; see /health" self.fixture.append( "nested path source", occurred_at_us=1_700_000_000_000_000, recorded_at_us=1_700_000_000_000_100, - context_note=secret, + context_note=note, ) - with self.assertRaises(Exception) as caught: - await self._operations()["workspace_export"]( - workspace=self.fixture.workspace, - request=_request("workspace_export", workspace_id=WORKSPACE_ID), - ) + bundle = await self._operations()["workspace_export"]( + workspace=self.fixture.workspace, + request=_request("workspace_export", workspace_id=WORKSPACE_ID), + ) - self.assertEqual("WORKSPACE_PATH_ESCAPE", caught.exception.code) - self.assertNotIn(secret, str(caught.exception)) + self.assertEqual( + {"note": note}, + bundle.events[0].payload["data"]["record"]["context"], + ) async def test_cancelled_sqlite_work_propagates_and_finishes_privately( self, diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py new file mode 100644 index 0000000..b5b9f61 --- /dev/null +++ b/tests/api_v7/test_path_policy.py @@ -0,0 +1,921 @@ +"""Path policy on the v7 wire (KD-2, UD-3). + +User text (memory content, rule text, queries, and server text quoting them) +may mention routes and file paths. Every other string keeps the +absolute-path rule, input and output, through one predicate. Anything the +server accepts must read back: nothing can be stored and then brick a read. +""" + +from __future__ import annotations + +import copy +import hashlib +import sqlite3 +import time +import types +from contextlib import closing +from pathlib import Path +from typing import Annotated, Any, Literal, Union, get_args, get_origin, get_type_hints + +import pytest +from fastmcp import Client +from pydantic import BaseModel, ValidationError, create_model + +from daem0nmcp.api.v7.errors import ErrorCode +from daem0nmcp.api.v7.models import ( + Cursor, + JsonObject, + RelativePath, + WireModel, + _user_text_fields, + is_host_absolute_path, +) +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.responses import ResponseFactory +from daem0nmcp.api.v7.tools import ( + TOOL_DATA_MODELS, + TOOL_INPUT_MODELS, + CodeIndexInput, + DiagnosticSummary, + DocumentIngestUrlInput, + ExportEvent, + HealthData, + HttpsUrl, + MediumText, + MemoryStoreInput, + PortableVectorPoint, + SandboxExecutionData, +) +from daem0nmcp.config import Settings +from daem0nmcp.discovery_projection import ( + CommunityProjectionSeed, + DiscoveryProjectionBuilder, + EntityProjectionSeed, + EntityRecordSeed, +) +from daem0nmcp.event_store import ( + EventCommand, + EventStore, + GovernanceEventCommand, + GovernanceEventStore, +) +from daem0nmcp.retrieval.runtime import await_projection_job_drains +from daem0nmcp.retrieval.specialized_projection import SpecializedProjectionBuilder +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces +from tests.api_v7.test_process_workspace_isolation_sweep import ( + CANARY, + CROSS_WORKSPACE_TOOLS, + PROFILE_ENVIRONMENT, + _code, + _is_protected, + _mint_tokens, + _seed_workspace_a, + _sweep_arguments, + _target_call, +) + +# The UD-3 examples: a regex escape, routes, and a Windows file location. +PATH_CONTENT = ("\\\\d+", "/health", "GET /api/v1/users", "C:\\proj\\src\\app.py") +# Mechanical-coverage seeds (R-26). +SEEDS = ("C:\\proj\\x.py", "/health", "\\d+", "\x1b[0m") +NEUTRAL = "plain neutral words" +# A path in text is the user's business; these outcomes mean it was refused. +# Readers turn a row that fails its output model into CAPABILITY_DEGRADED. +REFUSALS = frozenset( + { + "INVALID_ARGUMENT", + "INTERNAL_ERROR", + "WORKSPACE_PATH_ESCAPE", + "SCHEMA_REJECTED", + "CAPABILITY_DEGRADED", + } +) +RESOURCE_KINDS = ("warnings", "failures", "rules", "active-context") +TOKEN = "preflight-token-placeholder-0001" +WS = "ws_" + "0" * 24 + + +def _invoker(client: Client): + """Call a tool; a schema rejection by FastMCP becomes SCHEMA_REJECTED.""" + + async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await client.call_tool(tool, arguments, raise_on_error=False) + if result.structured_content is None: + assert result.is_error, tool + return {"ok": False, "error": {"code": "SCHEMA_REJECTED"}} + return result.structured_content + + return invoke + + +# --------------------------------------------------------------------------- +# Model-level checks. + + +def _store_input(**overrides: Any) -> dict[str, Any]: + return { + "workspace_id": WS, + "record_type": "decision", + "content": "text", + "idempotency_key": "path-policy-0001", + "preflight_token": TOKEN, + **overrides, + } + + +@pytest.mark.parametrize("path", ["C:\\x", "/etc/x"]) +def test_relative_file_path_rejects_host_paths(path): + with pytest.raises(ValidationError): + MemoryStoreInput.model_validate(_store_input(relative_file_path=path)) + + +@pytest.mark.parametrize("text", [*PATH_CONTENT, *SEEDS]) +def test_user_text_input_fields_accept_paths(text): + validated = MemoryStoreInput.model_validate( + _store_input( + content=text, + rationale=text, + tags=[text], + context={"note": text, text: [text]}, + ) + ) + assert validated.content == text + + +def test_server_generated_fields_reject_host_paths(): + with pytest.raises(ValidationError): + DiagnosticSummary(code="CODE_INDEX_SKIPPED", message="C:\\x") + health = { + "package_version": "7.0.0", + "protocol_version": "2025-06-18", + "supported_transports": {"stdio"}, + "auth_mode": "loopback", + "task_support": {"name": "tasks", "status": "ready"}, + } + HealthData.model_validate(health) + with pytest.raises(ValidationError): + HealthData.model_validate( + { + **health, + "capability_states": [ + { + "name": "storage", + "status": "failed", + "remediation": "Repair C:\\x", + } + ], + } + ) + + +def test_relative_paths_are_checked_by_their_own_validator(): + """Valid relative names and globs are not mistaken for absolute paths.""" + names = ["src/**/foo.py", "src/__pycache__/x", "docs-/a", "pkg_/a.py"] + index = CodeIndexInput(workspace_id=WS, patterns=names) + assert index.patterns == names + for invalid in ("C:\\x", "/etc/x", "../x", "src/../x", "~/x"): + with pytest.raises(ValidationError): + CodeIndexInput(workspace_id=WS, patterns=[invalid]) + + +def test_ingest_topic_accepts_paths(): + for text in SEEDS: + DocumentIngestUrlInput( + workspace_id=WS, + url="https://example.com/doc", + topic=text, + idempotency_key="path-policy-0001", + preflight_token=TOKEN, + ) + + +def test_error_remedies_echo_path_bearing_arguments(): + """A token error on path-bearing input returns its structured remedy (R-1).""" + for text in SEEDS: + failure = ( + ResponseFactory() + .begin(WS) + .failure( + ErrorCode.COUNSEL_REQUIRED, + "A preflight is required.", + remedy_tool="memory_preflight", + remedy_arguments={ + "workspace_id": WS, + "target_tool": "rule_create", + "target_arguments": {"trigger": text, "must_do": [text]}, + }, + ) + ) + assert failure.error.remedy.arguments["target_arguments"]["trigger"] == text + + +def test_sandbox_output_lines_may_mention_paths(): + line = 'File "/tmp/run.py", line 1, in ' + data = SandboxExecutionData( + success=False, + stdout="", + stderr=line, + exit_status=1, + execution_time_ms=5, + sanitized_logs=[f"stderr: {line}", "x" * 4096], + ) + assert data.sanitized_logs[0].endswith(line) + + +def test_export_structured_fields_stay_guarded(): + """Only user text in export payloads is exempt (R-3, R-4).""" + from daem0nmcp.api.v7.operations import CoreOperationError, _reject_raw_paths + + user = {"record": {"content": "/health", "context": {"note": "C:\\x"}}} + _reject_raw_paths([user]) + for relative in ("../../Users/x/y.py", "d:/other/x.py", "/etc/x"): + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path_relative": relative}}]) + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path": "C:\\proj\\x.py"}}]) + event = { + "event_id": "evt_" + "2" * 64, + "event_type": "memory.created", + "happened_at": "2026-08-08T12:00:00Z", + "content_hash": "a" * 64, + "payload": {"data": {"content": "C:\\proj\\x.py"}, "actor_id": "agent"}, + } + ExportEvent.model_validate(event) + with pytest.raises(ValidationError): + ExportEvent.model_validate( + {**event, "payload": {**event["payload"], "actor_id": "C:\\proj"}} + ) + with pytest.raises(ValidationError): + PortableVectorPoint(point_id="p", payload={"model_id": "C:\\m"}, vector=[1.0]) + + +def test_every_input_field_shape_is_classified(): + """Discovery walks every input model, bundles included, and knows each shape.""" + paths = { + f"{tool}.{'.'.join(map(str, path))}" + for tool, model in TOOL_INPUT_MODELS.items() + for path, _kind_ in _free_text_paths(model) + } + assert { + "workspace_import.bundle.events.0.payload", + "workspace_import.bundle.legacy_rows", + "memory_store_batch.records.0.context", + } <= paths, sorted(paths) + + +def _models(root: type[BaseModel], seen: set[type[BaseModel]]) -> None: + if root in seen: + return + seen.add(root) + for field in root.model_fields.values(): + for model in _nested_models(field.annotation): + _models(model, seen) + + +def _nested_models(annotation: object): + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + yield annotation + for argument in get_args(annotation): + yield from _nested_models(argument) + + +def _sample(annotation: object, text: str) -> object: + """A value of ``annotation``'s shape whose every string is ``text``.""" + origin = get_origin(annotation) + if origin is Annotated: + return _sample(get_args(annotation)[0], text) + if origin in (Union, types.UnionType): + options = [item for item in get_args(annotation) if item is not type(None)] + return _sample(options[0], text) + if origin in (list, set, frozenset): + return [_sample(get_args(annotation)[0], text)] + if origin is dict: + return {text: _sample(get_args(annotation)[1], text)} + if annotation is JsonObject: + return {"note": text, text: [text]} + assert annotation is str, annotation + return text + + +def test_user_text_round_trips_into_every_output_field_that_carries_it(): + """Each string a UserText input accepts validates in every UserText output. + + This shows the exemption works for every output field already typed + ``UserText`` (found from the models, each probe inheriting ``WireModel``'s + path check). It cannot show that the right fields carry the type; the + end-to-end test below does that. + """ + outputs: set[type[BaseModel]] = set() + for model in ( + *TOOL_DATA_MODELS.values(), + WarningResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + ActiveContextResourceDocument, + ): + _models(model, outputs) + carriers = { + (model.__name__, name): Annotated[ + (field.annotation, *field.metadata) # type: ignore[valid-type] + ] + if field.metadata + else field.annotation + for model in outputs + for name, field in model.model_fields.items() + if name in _user_text_fields(model) + } + assert len(carriers) > 30, sorted(carriers) + for text in (*PATH_CONTENT, *SEEDS): + MemoryStoreInput.model_validate(_store_input(content=text)) + for (model_name, name), annotation in carriers.items(): + probe = create_model( + f"{model_name}_{name}", __base__=WireModel, value=(annotation, ...) + ) + probe.model_validate({"value": _sample(annotation, text)}) + # The probe is meaningful: a guarded text field refuses the same text. + guarded = create_model("Guarded", __base__=WireModel, value=(MediumText, ...)) + if is_host_absolute_path(text): + with pytest.raises(ValidationError): + guarded.model_validate({"value": text}) + + +# --------------------------------------------------------------------------- +# Rows as a migrated or pre-fix database carries them. + + +def _hex(*parts: object) -> str: + return hashlib.sha256(repr(parts).encode()).hexdigest() + + +def _now_us() -> int: + return time.time_ns() // 1_000 + + +def _memory_record(record_type: str, text: str) -> dict[str, Any]: + return { + "record_type": record_type, + "legacy_type": None, + "content": f"Migrated note: {text}", + "rationale": text, + "context": {"note": text}, + "tags": [text], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "migration", + "source_model": None, + "deleted_at_us": None, + } + + +def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[str]: + """Write path-bearing records, a rule, a trigger, entities and a community.""" + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + record_ids: list[str] = [] + with closing(sqlite3.connect(database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + now = _now_us() - 1_000_000 + for index, text in enumerate(texts): + for record_type in ("decision", "warning"): + record_id = "mem_" + _hex("row", text, record_type) + EventStore(connection).append_and_project( + EventCommand( + workspace_id=workspace_id, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=now + index, + recorded_at_us=now + index, + actor_type="migration", + payload={"record": _memory_record(record_type, text)}, + ) + ) + record_ids.append(record_id) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="rule_" + _hex("rule", text), + stream_kind="rule", + event_type="rule.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "rule_id": "rule_" + _hex("rule", text), + "trigger": text, + "must_do": [text], + "must_not": [text], + "ask_first": [text], + "warnings": [text], + "priority": 0, + "enabled": True, + "created_at_us": now, + "updated_at_us": now, + }, + ) + ) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="trg_" + _hex("trigger", text), + stream_kind="trigger", + event_type="context_trigger.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "trigger_id": "trg_" + _hex("trigger", text), + # A file glob: a tag pattern is a regex, which + # "C:\proj\x.py" is not. + "trigger_type": "file", + "pattern": text, + "recall_query": text, + "categories": [], + "enabled": True, + "priority": 0, + "created_at_us": now, + "updated_at_us": now, + "deleted_at_us": None, + }, + ) + ) + connection.commit() + SpecializedProjectionBuilder(connection).rebuild(workspace_id, "graph") + DiscoveryProjectionBuilder(connection).populate_graph( + workspace_id, + entities=tuple( + EntityProjectionSeed( + name=text, + entity_type="file", + records=(EntityRecordSeed(record_ids[2 * index]),), + ) + for index, text in enumerate(texts) + if is_host_absolute_path(text) + ), + communities=( + CommunityProjectionSeed( + source_key="paths", + label=texts[0], + level=0, + member_record_ids=tuple(record_ids), + ), + ), + ) + # The graph rebuild queued by the events above just ran here; retire + # it so a later drain cannot swap the inserted entities for a new + # generation mid-test. + connection.execute( + "UPDATE background_jobs SET status='succeeded',updated_at_us=?," + "finished_at_us=? WHERE workspace_id=? AND status='queued' " + "AND idempotency_key='active-projection:graph'", + (_now_us(), _now_us(), workspace_id), + ) + connection.commit() + return record_ids + + +def _server(roots: tuple[Path, ...]): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(roots[0]), + workspace_roots=[str(root) for root in roots], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, + ) + + +async def test_path_content_is_stored_and_read_back(tmp_path): + """UD-3 content stores once and every read of it succeeds, pre-fix rows too.""" + root = tmp_path / "project" + (workspace,) = await initialize_workspaces((root,)) + ws = workspace.workspace_id + poisoned = _insert_rows(root, ws, ("\\\\d+",)) + async with Client(_server((root,))) as client: + invoke = _invoker(client) + + async def ok(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await _target_call(invoke, ws, tool, arguments) + assert result["ok"], (tool, arguments, result["error"]) + return result["data"] + + brief = await ok("session_brief", {}) + assert poisoned[1] in {item["record_id"] for item in brief["warnings"]} + for index, text in enumerate(PATH_CONTENT): + for record_type in ("decision", "warning"): + stored = await ok( + "memory_store", + { + "record_type": record_type, + "content": f"Remember {text}", + "idempotency_key": f"path-content-{index}-{record_type}", + }, + ) + record_id = stored["record"]["record_id"] + assert stored["record"]["excerpt"] == f"Remember {text}" + brief = await ok("session_brief", {}) + if record_type == "warning": + assert record_id in { + item["record_id"] for item in brief["warnings"] + } + await ok("memory_recall", {"query": text}) + await ok("memory_search_text", {"query": text}) + preflight = await invoke( + "memory_preflight", + { + "workspace_id": ws, + "target_tool": "memory_store", + "target_arguments": { + "record_type": record_type, + "content": text, + "idempotency_key": f"path-preflight-{index}", + }, + "description": text, + }, + ) + assert preflight["ok"], preflight["error"] + await ok( + "memory_record_outcome", + { + "record_id": record_id, + "outcome_text": text, + "worked": False, + "idempotency_key": f"path-outcome-{index}-{record_type}", + }, + ) + stale = await invoke( + "memory_store", + { + "workspace_id": ws, + "record_type": "decision", + "content": "GET /health returns 200", + "idempotency_key": "path-stale-token-0001", + "preflight_token": TOKEN, + }, + ) + assert stale["error"]["code"].startswith("TOKEN_"), stale + remedy = stale["error"]["remedy"] + assert remedy["tool"] == "memory_preflight" + assert remedy["arguments"]["target_arguments"]["content"] == ( + "GET /health returns 200" + ) + await ok("session_brief", {}) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{ws}/{kind}") + + +# --------------------------------------------------------------------------- +# Mechanical coverage (R-26). + +# An import bundle is hash-chained: editing any one string makes it +# IMPORT_INVALID. Its fields are covered by exporting the seeded workspace +# and importing it back (see the end of the test), not by field seeding. +ROUND_TRIPPED = frozenset({"workspace_import"}) +# The ingest URL is fetched before the topic is read, and tests have no +# network, so every call stops at the URL. The topic's input type is +# covered by test_ingest_topic_accepts_paths. +FETCH_FIRST = frozenset({"document_ingest_url.topic"}) + + +def _mentions_str(annotation: object) -> bool: + return annotation is str or any(_mentions_str(a) for a in get_args(annotation)) + + +def _kind(annotation: object) -> object: + """``"str"``, ``"json"``, a model class, ``("list", kind)`` or ``None``. + + Fails on any string-bearing shape it cannot classify, so a new free-text + shape cannot silently drop out of coverage. + """ + if annotation in (Cursor, RelativePath, HttpsUrl): + return None + origin = get_origin(annotation) + if origin in (Union, types.UnionType): + kinds = {_kind(item) for item in get_args(annotation) if item is not type(None)} + kinds.discard(None) + assert len(kinds) <= 1, f"mixed union {annotation!r}" + return kinds.pop() if kinds else None + if origin is Annotated: + base, *metadata = get_args(annotation) + if any(getattr(item, "pattern", None) for item in metadata): + return None + return _kind(base) + if origin in (list, set, frozenset): + inner = _kind(get_args(annotation)[0]) + assert not isinstance(inner, tuple), f"nested list {annotation!r}" + return None if inner is None else ("list", inner) + if origin is Literal: + return None + if annotation is str: + return "str" + if annotation is JsonObject: + return "json" + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation + assert not _mentions_str(annotation), f"unclassified {annotation!r}" + return None + + +def _free_text_paths(model: type[BaseModel], prefix=()): + """Every free-text field of ``model`` and its nested models, by annotation.""" + hints = get_type_hints(model, include_extras=True) + for name in model.model_fields: + kind = _kind(hints[name]) + if kind is None: + continue + if isinstance(kind, type): + yield from _free_text_paths(kind, (*prefix, name)) + elif isinstance(kind, tuple) and isinstance(kind[1], type): + yield from _free_text_paths(kind[1], (*prefix, name, 0)) + else: + yield (*prefix, name), kind + + +def _seed_value(kind: object, text: str) -> object: + if kind == "str": + return text + if kind == "json": + return {"note": text, text: [text]} + if kind == ("list", "str"): + return [text] + assert kind == ("list", "json"), kind + return [{"note": text, text: [text]}] + + +def _seeded(tool: str, base: dict[str, Any], path: tuple, kind: object, text: str): + """``base`` with the field at ``path`` set to ``text``, and its siblings fit.""" + arguments = copy.deepcopy(base) + target: Any = arguments + for step, following in zip(path[:-1], path[1:], strict=True): + if isinstance(step, int): + while len(target) <= step: + target.append({}) + elif step not in target: + target[step] = [] if isinstance(following, int) else {} + target = target[step] + name = path[-1] + # Exactly one selector is allowed; the seeded name replaces the id. + target.pop( + {"entity_name": "entity_id", "qualified_name": "code_entity_id"}.get(name, ""), + None, + ) + if name == "procedure_steps": + target["record_type"] = "procedure" + if (tool, path) == ("memory_preflight", ("target_arguments",)): + # Seed the target tool's own argument, not the envelope. + target["target_tool"] = "memory_store" + target["target_arguments"] = { + "record_type": "decision", + "content": text, + "idempotency_key": "mech-preflight-target-0001", + } + return arguments + target[name] = _seed_value(kind, text) + return arguments + + +def _fresh_keys(arguments: dict[str, Any], label: str) -> dict[str, Any]: + if "idempotency_key" in arguments: + digest = hashlib.sha256(label.encode()).hexdigest()[:40] + return {**arguments, "idempotency_key": f"mech-{digest}"} + return arguments + + +def _own_read_arguments(arguments: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in arguments.items() + if key not in {"cursor", "after_cursor"} + } + + +async def _seed_free_text(invoke, workspace_id: str, tool: str, arguments: dict): + """Call ``tool`` with each free-text field set to each seed. + + The neutral-text call must not be refused, so a wrongly guarded field + cannot hide behind bad base arguments. (It may miss: a lookup by a name + that does not exist is NOT_FOUND.) Each seeded call must then get the + neutral call's outcome or at least not a refusal. + """ + if tool in ROUND_TRIPPED: + return [] + base = _own_read_arguments(arguments) + labels = [] + for path, kind in _free_text_paths(TOOL_INPUT_MODELS[tool]): + label = f"{tool}.{'.'.join(map(str, path))}" + if label in FETCH_FIRST: + continue + labels.append(label) + codes = [] + for text in (NEUTRAL, *SEEDS): + call = _fresh_keys(_seeded(tool, base, path, kind, text), label + text) + response = await _target_call(invoke, workspace_id, tool, call) + if _code(response) == "INTERNAL_ERROR": + # SQLite contention with a background projection drain is still + # reported as INTERNAL_ERROR on this base (PR 3 makes it + # retryable). A refusal caused by the text repeats; retry once. + call = _fresh_keys(call, label + text + "retry") + response = await _target_call(invoke, workspace_id, tool, call) + codes.append(_code(response)) + neutral, *seeded = codes + assert neutral not in REFUSALS, (label, neutral) + for text, code in zip(SEEDS, seeded, strict=True): + assert code == neutral or code not in REFUSALS, (label, text, code, neutral) + return labels + + +def _is_write(tool: str) -> bool: + # memory_record_outcome writes a memory event without a preflight; like + # every write it marks the graph stale until the next rebuild. + return _is_protected(tool) or tool == "memory_record_outcome" + + +async def _read_everything(invoke, workspace_id: str, seed) -> dict[str, str]: + """Every read tool with the workspace's own objects.""" + codes: dict[str, str] = {} + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if _is_write(tool) or tool in CROSS_WORKSPACE_TOOLS: + continue + plain = _fresh_keys(_own_read_arguments(arguments), f"{tool}-{len(codes)}") + codes[tool] = _code(await invoke(tool, {"workspace_id": workspace_id, **plain})) + return codes + + +async def _export_pages(invoke, workspace_id: str) -> list[dict[str, Any]]: + scope = {"workspace_id": workspace_id, "page_byte_limit": 65536} + page = await invoke("workspace_export", scope) + assert page["ok"], page["error"] + pages = [page["data"]] + while not pages[-1]["complete"]: + previous = pages[-1] + page = await invoke( + "workspace_export", + { + **scope, + "export_session_id": previous["export_session_id"], + "page_index": previous["page_index"] + 1, + "cursor": previous["next_cursor"], + }, + ) + assert page["ok"], page["error"] + pages.append(page["data"]) + return pages + + +async def _import_pages(invoke, workspace_id: str, pages) -> dict[str, Any]: + import_id = None + key = "mech-import-0001" + for page in pages: + arguments = {"bundle": page, "finalize": False, "idempotency_key": key} + if import_id is not None: + arguments["import_session_id"] = import_id + staged = await _target_call(invoke, workspace_id, "workspace_import", arguments) + assert staged["ok"], staged["error"] + import_id = staged["data"]["import_session_id"] + finish = {"import_session_id": import_id, "finalize": True, "idempotency_key": key} + done = await _target_call(invoke, workspace_id, "workspace_import", finish) + assert done["ok"], done["error"] + return done["data"] + + +def _texts(value: Any) -> set[str]: + if isinstance(value, dict): + return {text for item in value.values() for text in _texts(item)} + if isinstance(value, list): + return {text for item in value for text in _texts(item)} + return {value} if isinstance(value, str) else set() + + +# Reads that stay gated whatever is stored; every other read must be ok. +GATED_READS = { + # Needs a native edit request, which only the edit hook makes. + "edit_preflight": "NOT_FOUND", + # The default selector exceeds the foreground node budget; the same + # tools are read below with a small selector over the path records. + "knowledge_graph_get": "TASK_REQUIRED", + "knowledge_graph_render": "TASK_REQUIRED", +} + + +async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_path): + """Seed every free-text input field and DB row with paths; read everything. + + An input field typed as free text but not ``UserText`` refuses the seed, + and an output field carrying it fails or degrades the read; either fails + this test. The workspace is then exported and imported back. + """ + a_root, b_root = tmp_path / "alpha", tmp_path / "beta" + alpha, beta = await initialize_workspaces((a_root, b_root)) + a, b = alpha.workspace_id, beta.workspace_id + (a_root / "src").mkdir() + (a_root / "src" / "owned.py").write_text( + f"def {CANARY}_function():\n return 1 # TODO: see C:\\proj\\x.py /health\n", + encoding="utf-8", + ) + seeded: list[str] = [] + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + seed = await _seed_workspace_a(invoke, a, b) + await _mint_tokens(invoke, a, b, seed) + # Writes first; reads are seeded below, once every kind of row exists + # (a name lookup before any entity exists stops at the graph gate). + writes = { + tool: arguments + for tool, arguments in _sweep_arguments(seed).items() + if _is_write(tool) + } + writes["workspace_link"] = {"linked_workspace_id": b} + for tool, arguments in sorted(writes.items()): + seeded += await _seed_free_text(invoke, a, tool, arguments) + + # Rows as a migrated v6 database or a pre-fix write left them. + record_ids = _insert_rows(a_root, a, SEEDS) + + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + await _mint_tokens(invoke, a, b, seed) + entities = await invoke("entity_list", {"workspace_id": a, "limit": 100}) + path_seeds = {text for text in SEEDS if is_host_absolute_path(text)} + assert path_seeds <= {item["name"] for item in entities["data"]["items"]} + seed.ids["entity"] = entities["data"]["items"][0]["entity_id"] + # The inserted graph generation replaces the seed's community IDs. + communities = await invoke("community_list", {"workspace_id": a}) + seed.ids["community"] = communities["data"]["items"][0]["community_id"] + # The stored text comes back verbatim, not dropped or degraded. + # Before any read that writes (outcomes, rebuilds) makes the graph stale. + shown: set[str] = set() + for tool, arguments in ( + ("rule_list", {"limit": 100}), + ("context_trigger_list", {"limit": 100}), + ("memory_search_text", {"query": "Migrated note", "limit": 100}), + ( + "knowledge_graph_render", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ( + "knowledge_graph_get", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ): + response = await invoke(tool, {"workspace_id": a, **arguments}) + assert response["ok"], (tool, response["error"]) + shown |= _texts(response["data"]) + assert set(SEEDS) <= shown, set(SEEDS) - shown + assert {f"Migrated note: {text}" for text in SEEDS} <= shown + after = await _read_everything(invoke, a, seed) + assert {t: c for t, c in after.items() if c != "ok"} == GATED_READS, after + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if tool in after: + # Query-like fields searching for the paths. + seeded += await _seed_free_text(invoke, a, tool, arguments) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{a}/{kind}") + # Let background projection drains settle so the export pages form + # one consistent snapshot. + await await_projection_job_drains() + pages = await _export_pages(invoke, a) + + # The exported workspace imports back into empty storage (R-24). + storage = a_root / ".daem0nmcp" / "storage" + storage.rename(a_root / ".daem0nmcp" / "retained-before-import") + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + restored = await _import_pages(invoke, a, pages) + assert restored["root_hash"] == pages[0]["root_hash"] + found = await invoke( + "memory_search_text", + {"workspace_id": a, "query": "Migrated note", "limit": 100}, + ) + assert found["ok"], found["error"] + assert {f"Migrated note: {text}" for text in SEEDS} <= _texts(found["data"]) + + # Nested and cross-workspace inputs are reached too; the floor catches a + # change that quietly drops fields from coverage. + assert { + "memory_store.context", + "memory_store.procedure_steps", + "memory_store_batch.records.0.content", + "memory_preflight.target_arguments", + "entity_evolution_trace.entity_name", + "rule_update.patch.must_do", + "workspace_link.label", + } <= set(seeded), seeded + assert len(seeded) >= 60, len(seeded) diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index 6900624..c72d5a5 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -109,13 +109,29 @@ async def guidance(self, *args: object, **kwargs: object) -> object: class _LeakyPreflightService: - async def guidance(self, *args: object, **kwargs: object) -> object: - from daem0nmcp.api.v7.tools import PreflightGuidance + """Guidance whose structured path field holds a host path. + + The rule warning also names a path, which user text may do (UD-3); only + ``relative_file_path`` makes the guidance invalid. + """ + async def guidance(self, *args: object, **kwargs: object) -> object: del args, kwargs - return PreflightGuidance( - warnings=[r"Inspect D:\private\workspace\policy.txt"], - ) + return { + "warnings": [r"Inspect D:\private\workspace\notes.txt"], + "records": [ + { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": "Guarded path field.", + "relative_file_path": r"D:\private\workspace\policy.txt", + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + ], + } class _ExplodingRecallService: @@ -1000,7 +1016,7 @@ def test_failed_preflight_guidance_is_opaque_and_issues_no_token(self) -> None: self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) def test_path_bearing_guidance_fails_before_capability_issue(self) -> None: - # Catches issuing a usable token for a response that must be redacted. + # Catches issuing a usable token for guidance whose path field is invalid. from daem0nmcp.api.v7.errors import ErrorCode from daem0nmcp.api.v7.pinned import build_pinned_handlers from daem0nmcp.api.v7.tools import MemoryPreflightOutput diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 93ea35a..c180ca2 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -13,8 +13,11 @@ No target response may carry A's text, A's canary token, A's IDs or A's workspace ID; A's database may not change; a rejected call may not change the -target; and no string or key anywhere may carry a registered root or the home -directory. Every call's outcome is pinned. +target; and no string or key outside a ``UserText`` field may carry a +registered root or the home directory. Each target stores a note naming its +own root, which user text may do (UD-3), so the fields that echo it are found +from the same response models the server validates against, never by field +name. Every call's outcome is pinned. Pass 2 relaunches A and B over HTTP with a JWT principal granted only A. Every call that targets B, directly or as a consolidation or federation @@ -38,9 +41,18 @@ import pytest from fastmcp import Client from mcp.shared.exceptions import McpError +from pydantic import BaseModel, ValidationError +from daem0nmcp.api.v7.models import ApiResponse, RecordSummary, guarded_strings from daem0nmcp.api.v7.policy import V7_TOOL_LEVELS from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.tools import TOOL_DATA_MODELS, DiagnosticSummary from daem0nmcp.config import Settings from daem0nmcp.covenant import CovenantLevel from daem0nmcp.storage_activation import resolve_active_database @@ -94,6 +106,43 @@ Invoke = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] +RESOURCE_MODELS: dict[str, type[BaseModel]] = { + "warnings": WarningResourceDocument, + "failures": FailureResourceDocument, + "rules": RuleResourceDocument, + "active-context": ActiveContextResourceDocument, +} + + +def _response_model(tool: str) -> type[BaseModel]: + return ApiResponse[TOOL_DATA_MODELS[tool]] # type: ignore[name-defined] + + +# Marks the one piece of user text allowed to name a host path: the note each +# target stores about its own root. +NOTE_MARKER = "sweepnote4b1d" + + +def _host_path_leaks( + model: type[BaseModel], document: Any, needles: set[str] +) -> list[str]: + """Strings that contain a host path needle, other than the stored note. + + Every string is scanned, UserText fields included, so server-assembled + user-text fields (excerpts, rendered context, TODO text, mermaid) stay + watched. A hit is allowed only in a UserText field whose text carries the + note's marker. Validating against the response model also refuses any + absolute path in a server-generated field, whatever its spelling. + """ + validated = model.model_validate(document, strict=False) + guarded = set(guarded_strings(validated)) + return [ + value + for value in _strings(document) + if any(needle in value.lower() for needle in needles) + and (value in guarded or NOTE_MARKER not in value) + ] + def _placeholder(prefix: str) -> str: return f"{prefix}_{hashlib.sha256(prefix.encode()).hexdigest()}" @@ -641,6 +690,36 @@ def _is_protected(tool: str) -> bool: return V7_TOOL_LEVELS[tool] in {CovenantLevel.COUNSEL, CovenantLevel.DESTRUCTIVE} +def test_leak_check_exempts_user_text_by_annotation_only(tmp_path): + needles = {str(tmp_path).lower(), "alpha-private-root"} + summary = { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": f"Target note {NOTE_MARKER}: see {tmp_path}", + "tags": ["/health"], + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + assert _host_path_leaks(RecordSummary, summary, needles) == [] + # A root in user text the server assembled on its own is still caught. + unmarked = {**summary, "excerpt": f"Rendered from {tmp_path}"} + assert _host_path_leaks(RecordSummary, unmarked, needles) == [ + f"Rendered from {tmp_path}" + ] + # The same root in a server-generated field is refused by the model... + with pytest.raises(ValidationError): + _host_path_leaks( + DiagnosticSummary, {"code": "X_Y", "message": f"at {tmp_path}"}, needles + ) + # ...and a spelling the path rule does not see is found by the scan. + leaked = {"code": "X_Y", "message": "at alpha-private-root"} + assert _host_path_leaks(DiagnosticSummary, leaked, needles) == [ + "at alpha-private-root" + ] + + def test_every_manifest_tool_is_swept_or_declared_cross_workspace(): swept = set(_sweep_arguments(Seed())) cross = set(CROSS_WORKSPACE_TOOLS) @@ -666,7 +745,9 @@ def __init__(self, roots: dict[str, Path], a_root: Path, seed: Seed, a: str): self.needles = needles | {Path(n).as_posix().lower() for n in needles} self.observed: dict[str, dict[str, list[str]]] = {} - def check(self, label: str, response: Any, request: Any) -> None: + def check( + self, label: str, response: Any, request: Any, model: type[BaseModel] + ) -> None: echoed = set(_strings(request)) for value in _strings(response): for secret in self.forbidden: @@ -674,19 +755,31 @@ def check(self, label: str, response: Any, request: Any) -> None: # A bare canary may appear only as an exact echo of the request # (memory_verify returns its claim). assert CANARY not in value or value in echoed, f"{label}: {value!r}" - lowered = value.lower() - for needle in self.needles: - assert needle not in lowered, f"{label}: host path in {value!r}" + leaks = _host_path_leaks(model, response, self.needles) + assert not leaks, f"{label}: host path in {leaks!r}" assert _ledger_counts(self.a_root) == self.a_ledger, f"{label} changed A" async def run(self, client, name: str, target: str, root: Path) -> None: invoke = _invoker(client) observed = self.observed.setdefault(name, {}) assert (await invoke("session_brief", {"workspace_id": target}))["ok"] + # The target's own note names its root. Reads echo it only in + # UserText fields; the leak check must pass without a name exemption. + # The marker is in the file name too, so a file entity the graph + # extracts from the note (POSIX paths only) carries it as well. + marked = root / "src" / f"{NOTE_MARKER}.py" + note = { + "record_type": "warning", + "content": f"Target note {NOTE_MARKER}: see {marked}", + "idempotency_key": f"sweep-{name}-host-note-0001", + } + assert (await _protected(invoke, target, "memory_store", note))["ok"] own: dict[str, str] = {} for tool, arguments in OWN_VARIANTS: response = await _target_call(invoke, target, tool, arguments) - self.check(f"{name}:{tool} (own)", response, arguments) + self.check( + f"{name}:{tool} (own)", response, arguments, _response_model(tool) + ) own[tool] = _code(response) if tool == "code_search" and APPS_AVAILABLE: # The scan must cover real results, not an empty page. @@ -705,19 +798,16 @@ async def run(self, client, name: str, target: str, root: Path) -> None: for variant in variants: before = _ledger_counts(root) response = await _target_call(invoke, target, tool, variant) - self.check(f"{name}:{tool}", response, variant) + self.check(f"{name}:{tool}", response, variant, _response_model(tool)) if not response["ok"]: assert _ledger_counts(root) == before, f"{name}:{tool} wrote" observed.setdefault(tool, []).append(_code(response)) for tool, code in own.items(): observed[tool].append(code) - templates = [ - f"memory://workspaces/{target}/{kind}" - for kind in ("warnings", "failures", "rules", "active-context") - ] - for uri in templates: - contents = await client.read_resource(uri) - self.check(uri, [json.loads(item.text) for item in contents], uri) + for kind, model in RESOURCE_MODELS.items(): + uri = f"memory://workspaces/{target}/{kind}" + for item in await client.read_resource(uri): + self.check(uri, json.loads(item.text), uri, model) def assert_pinned(self, name: str) -> None: for tool, codes in sorted(self.observed[name].items()): @@ -802,7 +892,7 @@ async def _cross_workspace_checks(invoke, sweep, a, b, b_root, seed) -> list[str async def refused(tool, arguments, code): before = _ledger_counts(b_root) response = await _target_call(invoke, b, tool, arguments) - sweep.check(f"B:{tool}", response, arguments) + sweep.check(f"B:{tool}", response, arguments, _response_model(tool)) assert _code(response) == code, (tool, response) assert _ledger_counts(b_root) == before, tool diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index 45a43a4..b97fab7 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -469,22 +469,21 @@ async def test_memory_flags_are_honored_without_losing_row_state(self) -> None: self.assertEqual(by_id[deleted].item.current_status, "invalidated") self.assertTrue(by_id[deleted].deleted) - async def test_canonical_root_embedded_in_public_text_fails_closed(self) -> None: - # Catches free-text fields bypassing the path-safe structured field policy. - self.fixture.add_record( - 12, - content=f"do not expose {self.fixture.workspace_root / 'secret.py'}", - ) + async def test_user_text_naming_the_workspace_root_reads_back(self) -> None: + # User text may mention paths, the root included (UD-3); one such + # warning must not make the resource unreadable. + content = f"see {self.fixture.workspace_root / 'app.py'}" + record_id = self.fixture.add_record(12, content=content) repository = self._repository() - from daem0nmcp.api.v7.resource_repository import ResourceRepositoryError - with self.assertRaises(ResourceRepositoryError): - await repository.read_warnings( - self.fixture.workspace, - ResourceReadRequest( - kind="warnings", limit=1, order_by="updated_at_desc" - ), - ) + rows = await repository.read_warnings( + self.fixture.workspace, + ResourceReadRequest(kind="warnings", limit=1, order_by="updated_at_desc"), + ) + + self.assertEqual( + [(record_id, content)], [(r.item.record_id, r.item.excerpt) for r in rows] + ) async def test_rules_use_canonical_ids_and_highest_priority_enabled_order( self, diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index 7f31a5f..ffffc8d 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -1239,7 +1239,7 @@ async def leaky_reader(_workspace, _request): with self.assertRaisesRegex( RuntimeServiceError, - "UNSAFE_SERVICE_OUTPUT", + "BRIEFING_FAILED", ) as leak: await BasicBriefingService(reader=leaky_reader).assemble( self.workspace, diff --git a/tests/test_memory.py b/tests/test_memory.py index 532269b..9f4ff41 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -306,8 +306,15 @@ async def test_v7_legacy_id_keeps_opaque_canonical_origin(self, memory_manager): assert selected["record_id"].startswith("mem_") assert selected["evidence_refs"][0]["record_id"] == selected["record_id"] assert selected["origin_workspace_id"].startswith("ws_") - # Public absolute paths are never re-exposed through legacy context. - assert selected["context"] is None + # Context is user text and may mention paths (UD-3); it comes back as + # stored, while the provenance fields stay opaque and path-free. + assert selected["context"] == {"reason": r"stored beside C:\private\source.py"} + provenance = [ + selected["record_id"], + selected["origin_workspace_id"], + *map(str, selected["evidence_refs"]), + ] + assert not any("private" in value for value in provenance), provenance @pytest.mark.asyncio async def test_failed_decisions_boosted(self, memory_manager): From edfaa5752a242a866fefbeeb63acf7ba423fd2fc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 20 Sep 2026 18:57:17 +0000 Subject: [PATCH 68/72] fix: pass an empty warning sequence for tokenless preflight response.success expects Sequence[ApiWarning], not None. The merge replaced _path_safe_success, which previously coerced None to (). Co-authored-by: Devil <9thLevelSoftware@users.noreply.github.com> --- daem0nmcp/api/v7/pinned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 711c69c..41e9aae 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -574,7 +574,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=None, ), - warnings=[draft_warning] if draft_warning is not None else None, + warnings=([draft_warning] if draft_warning is not None else ()), ) token = self._dependencies.covenant_gate.issue_preflight( scope, From cd5842f893dbcc5d1488faa69fa8587dab1dd5f4 Mon Sep 17 00:00:00 2001 From: Devil <69057727+9thLevelSoftware@users.noreply.github.com> Date: Sun, 20 Sep 2026 14:59:22 -0400 Subject: [PATCH 69/72] PR 3/12: Report transient failures as retryable and log every internal error (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Brownfield run 3403b448, PR 3 of 12** — audit-and-fix stack for the v7 core (#61/#62). Base: `v7/bf-01-make-ci-green-and-pin-workspace`. ## Plan (contract) ### PR 3: Report transient failures as retryable and log every internal error - Depends on: PR 1 - Files: daem0nmcp/api/v7/responses.py, daem0nmcp/api/v7/runtime_services.py, daem0nmcp/api/v7/resource_repository.py, daem0nmcp/api/v7/pinned.py, daem0nmcp/event_store.py, daem0nmcp/api/v7/errors.py, tests/api_v7/test_error_honesty.py, tests/api_v7/test_process_concurrent_writers.py, tests/api_v7/test_pinned_handlers.py - Traces to: FP-2, F-020, F-027, F-021, F-028, F-055, F-066, F-070, F-082, A-011 - Risk: low - Description (KD-6): 1. **Busy mapping.** In `SQLiteMemoryEventWriter._run_mutation` and the outcome path (`runtime_services.py:570-597, 757-770, 883-886`) and in `ResourceRepository._run` (`resource_repository.py:947-984`), map `BoundedWorkerBusyError` and `sqlite3.OperationalError` "database is locked"/"busy" to `DATABASE_IN_USE` (retryable, `retry_after_ms=250`) in `pinned._EXPECTED_SERVICE_ERRORS`. 2. **Logging.** In `ResponseContext.internal_error`, replace `del error` with `logger.exception("v7 internal error correlation_id=%s", …, exc_info=error)` to stderr. Service raises chain `from exc`. The wire envelope is unchanged. 3. **Conflict mapping.** In `event_store.py:539-542`, only `sqlite3.IntegrityError` becomes `EventStreamConflict` (F-066). 4. **Null token.** `memory_preflight` with target arguments that don't normalize keeps `ok:true`, but adds a `meta.warnings` entry with the field errors and "no preflight token issued" (F-070/F-082). 5. **Tests.** - 8 parallel preflight+store flows: every response is either ok or retryable `DATABASE_IN_USE`, and after retries there are exactly 8 records and 8 events. - A forced lock returns `DATABASE_IN_USE`. - An induced exception is logged with the correlation ID returned to the client. - Two `process_client` stdio servers on one workspace fire 5 same-key and 10 distinct-key stores concurrently: exactly one event per key and no `INTERNAL_ERROR` (F-055). - Acceptance: - The new tests pass on 3 OSes. - There is no `del error` left in `responses.py`. --- ## What landed # PR 3 summary: Report transient failures as retryable and log every internal error (Saved by the orchestrator from the implementer's reply.) Branch `v7/bf-03-report-transient-failures-as-ret`, base f7f024a (PR 1 tip). Worktree `D:\Daem0n-MCP\.claude\worktrees\agent-a0bb29ebdbe88c9ac`. - 358256f `fix: report only SQLite integrity errors as event stream conflicts` - 83a1b7b `fix: report transient contention as retryable and log every internal error` (final) ## What changed 1. **Busy mapping (F-020, F-027, A-011)** - `api/v7/errors.py`: `is_database_busy(error)` — true for `BoundedWorkerBusyError`, `code == "DATABASE_IN_USE"`, or `sqlite3.OperationalError` containing "locked"/"busy". - `runtime_services.py`: `SQLiteMemoryEventWriter._run_mutation` maps `BoundedWorkerBusyError` → `RuntimeServiceError("DATABASE_IN_USE")`; `_store_sync`/`_record_outcome_sync` catch-alls map busy → DATABASE_IN_USE (else MEMORY_STORE_FAILED/MEMORY_OUTCOME_FAILED); `_open_database` busy → DATABASE_IN_USE (else ACTIVE_V7_UNAVAILABLE); `BasicBriefingService.assemble` and `BasicPreflightService.guidance` pass DATABASE_IN_USE through instead of BRIEFING_FAILED/PREFLIGHT_FAILED. - `resource_repository.py`: `ResourceRepositoryError` optional `code`; `_run` raises `ResourceRepositoryError("DATABASE_IN_USE")` when busy; always chains the cause. - `pinned.py`: `_expected_service_failure` adds `retry_after_ms=250` for DATABASE_IN_USE (already retryable). 2. **Logging (F-021, F-028)** - `responses.py`: `del error` removed; `internal_error` logs `"v7 internal error correlation_id=%s"` with `exc_info` (error or active exception) via `logging.getLogger(__name__).error`. Goes to stderr (last-resort handler / v6 basicConfig), never the MCP wire; wire envelope unchanged; no "NoneType: None" when no exception. - `runtime_services.py`: all 28 `raise RuntimeServiceError(...) from None` → `from exc`. MCP-facing resource/middleware errors keep `from None` (tests assert `__cause__ is None`). 3. **Conflict mapping (F-066)**: `event_store.py` INSERT into `memory_events` maps only `sqlite3.IntegrityError` → `EventStreamConflict`; I/O and lock errors propagate. aiosqlite path raises the same class. 4. **Tokenless preflight (F-070, F-082)**: still `ok:true` with null token, now plus `meta.warnings` `PREFLIGHT_TOKEN_NOT_ISSUED` listing schema field names and pydantic error types only (unknown keys as `?`, values never echoed). `_path_safe_success` gained optional `warnings`. ## Tests - New `tests/api_v7/test_error_honesty.py` (in-process `create_v7_server("stdio")` + `fastmcp.Client`): 8 parallel preflight+store flows → each ok or retryable DATABASE_IN_USE with 250 ms; after retries exactly 8 records and 8 `memory.created` (13–14 busy retries per run locally). Held `BEGIN IMMEDIATE` → store returns retryable DATABASE_IN_USE after 5 s busy timeout; retry succeeds after release. Induced failure in writer transaction → INTERNAL_ERROR; response lacks the canary; log record contains the returned correlation_id and traceback shows canary + MEMORY_STORE_FAILED. Full resource-repository pool → `code == "DATABASE_IN_USE"`. - New `tests/api_v7/test_process_concurrent_writers.py` (F-055): two real stdio servers on one workspace; 5 same-key + 10 distinct-key concurrent stores → one `memory.created` per key, 11 records, exactly one non-replay among same-key, no code except DATABASE_IN_USE; bounded 120 s deadline honouring `retry_after_ms`. - `test_pinned_handlers.py`: description-only preflight checks the warning; new DATABASE_IN_USE → retryable, 250 ms. - `test_runtime_services.py`: cancelled-worker test now expects `RuntimeServiceError("DATABASE_IN_USE")` caused by `BoundedWorkerBusyError`. - `test_event_store.py`: `test_only_integrity_errors_on_insert_are_stream_conflicts`. - Reverted-source check: all 8 new/changed tests failed on old code; two-process test reproduced F-055 (INTERNAL_ERROR). - Full suite Windows 3.12: 2809 passed, 121 skipped, 0 failed (834 s). New tests 5× in a row: all pass (~21 s each). Linux Docker python:3.10-slim affected modules: 94 passed. ruff check/format clean; mypy api/v7 0 errors on win32/linux/darwin; release inventory valid. Sweep EXPECTED pins unchanged and green. macOS unverified until CI. ## Deviations 1. `tests/api_v7/process_client.py` gained `log_name` (default unchanged) so two servers don't share `stdio-server.log`. 2. IntegrityError test in `tests/test_event_store.py` (not in file list) to reuse its schema fixture. 3. DATABASE_IN_USE also flows through `_open_database` and briefing/preflight services, beyond the plan's line ranges — needed for contended session_brief/preflight (F-020's 8 concurrent briefs). 4. `logger.error(..., exc_info=...)` instead of `logger.exception` (some call sites aren't in an except block). 5. Two commits rather than one per item. ## Follow-ups (not fixed) - `application.py:286-300`: non-pinned tools send any registered error code, incl. DATABASE_IN_USE, with `retryable=False` — contention on non-pinned tools still non-retryable. One-line fix, outside file list. - `local_state_operations.py` has its own busy check (`_translate_error`) and maps a full pool to TASK_REQUIRED; could use `is_database_busy`/DATABASE_IN_USE. - Other pools map `BoundedWorkerBusyError` to inconsistent codes (F-071); recall maps full pool to retryable RETRIEVAL_UNAVAILABLE (left as is). - `.gitignore` lacks `.venv/`. ## Round 2 ce94143 (overrun-then-clean reads → retryable DATABASE_IN_USE; application.py non-pinned DATABASE_IN_USE retryable + remedy; busy memory_store carries memory_preflight remedy; sqlite_errorcode-based busy detection on 3.11+; governance_events IntegrityError-only; ValidationError logged without input values; locked_current storage lock → DATABASE_IN_USE; "workspace unavailable" responses logged with cause) and 987bd00 (root cause of preflight timeouts: guidance read no longer spawns git — `read_briefing_snapshot(include_git_changes=False)`; removed from PR 4's plan). 14 addressed, 2 wontfix (R-8 branch is reachable; R-16 deferred with F-071). Tests reviewer verified R-9..R-12 (20/20 + 12/12 loaded runs). CI run 35423993350 green after re-run of one unrelated portability flake (carried to PR 6). --- ## Review # PR 3 review — merged (round 2: all addressed; tests reviewer verified; CI 35423993350 green after flake re-run) Reviewer slots: general, general-2, tests, security, plan-alignment (effort 5). Item statuses: 14 addressed, 2 wontfix. CI: https://github.com/9thlevelsoftware/Daem0n-MCP/actions/runs/35423993350 (green) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- daem0nmcp/api/v7/application.py | 48 +- daem0nmcp/api/v7/discovery_operations.py | 66 +- daem0nmcp/api/v7/errors.py | 36 ++ daem0nmcp/api/v7/operations.py | 2 + daem0nmcp/api/v7/pinned.py | 99 ++- daem0nmcp/api/v7/production.py | 1 + daem0nmcp/api/v7/resource_repository.py | 34 +- daem0nmcp/api/v7/responses.py | 77 ++- daem0nmcp/api/v7/runtime_services.py | 123 ++-- daem0nmcp/cli.py | 33 +- daem0nmcp/event_store.py | 9 +- daem0nmcp/retrieval/jobs.py | 39 +- daem0nmcp/retrieval/lexical_config.py | 9 + daem0nmcp/retrieval/operations.py | 80 ++- daem0nmcp/retrieval/projections.py | 159 +++++ daem0nmcp/retrieval/specialized_projection.py | 15 +- daem0nmcp/verification_v7.py | 46 +- docs/release/v7/inventory.md | 3 +- docs/release/v7/requirements.json | 14 +- tests/api_v7/process_client.py | 3 +- tests/api_v7/test_application.py | 53 ++ tests/api_v7/test_discovery_operations.py | 79 +++ tests/api_v7/test_error_honesty.py | 279 +++++++++ tests/api_v7/test_pinned_handlers.py | 56 ++ .../api_v7/test_process_concurrent_writers.py | 117 ++++ .../api_v7/test_process_discovery_surface.py | 30 +- tests/api_v7/test_production.py | 51 ++ tests/api_v7/test_resource_repository.py | 30 +- tests/api_v7/test_runtime_services.py | 5 +- tests/test_cli_retrieval.py | 61 ++ tests/test_event_store.py | 97 +++ tests/test_migrate_v7.py | 3 +- tests/test_retrieval_projection_growth.py | 570 ++++++++++++++++++ tests/test_verify_v7.py | 27 + 34 files changed, 2218 insertions(+), 136 deletions(-) create mode 100644 tests/api_v7/test_error_honesty.py create mode 100644 tests/api_v7/test_process_concurrent_writers.py create mode 100644 tests/test_retrieval_projection_growth.py diff --git a/daem0nmcp/api/v7/application.py b/daem0nmcp/api/v7/application.py index 2104068..75a76f3 100644 --- a/daem0nmcp/api/v7/application.py +++ b/daem0nmcp/api/v7/application.py @@ -15,7 +15,7 @@ from ...covenant import CovenantGate, InvocationScope from ...workspace import Workspace -from .errors import ErrorCode +from .errors import DATABASE_IN_USE_RETRY_AFTER_MS, ErrorCode from .models import CapabilityState from .policy import V7_TOOL_LEVELS from .responses import ResponseFactory @@ -119,18 +119,26 @@ def _remedy( if code in {"COUNSEL_REQUIRED", "TOKEN_MISSING"} and isinstance( workspace_id, str ): - target_arguments = { - key: value - for key, value in effective.items() - if key not in {"workspace_id", "preflight_token"} - } - return "memory_preflight", { - "workspace_id": workspace_id, - "target_tool": tool_name, - "target_arguments": target_arguments, - } + return self._preflight_remedy(workspace_id, tool_name, effective) return None, {} + @staticmethod + def _preflight_remedy( + workspace_id: str, + tool_name: str, + effective: Mapping[str, Any], + ) -> tuple[str, dict[str, Any]]: + target_arguments = { + key: value + for key, value in effective.items() + if key not in {"workspace_id", "preflight_token"} + } + return "memory_preflight", { + "workspace_id": workspace_id, + "target_tool": tool_name, + "target_arguments": target_arguments, + } + def handler(self, tool_name: str) -> Callable[..., Any]: try: input_model = TOOL_INPUT_MODELS[tool_name] @@ -294,6 +302,24 @@ async def invoke(**arguments: Any) -> object: states = () if any(not isinstance(state, CapabilityState) for state in states): states = () + if error_code == ErrorCode.DATABASE_IN_USE.value: + # The gate already spent any preflight token, so a + # retry needs a fresh one. + workspace_id = effective.get("workspace_id") + remedy_tool, remedy_arguments = ( + self._preflight_remedy(workspace_id, tool_name, effective) + if isinstance(token, str) and isinstance(workspace_id, str) + else (None, {}) + ) + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool=remedy_tool, + remedy_arguments=remedy_arguments, + capability_states=tuple(states), + ) return response.failure( ErrorCode(error_code), "The operation could not be completed.", diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 009e4d2..62858f1 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -2027,6 +2027,56 @@ def reader(connection: sqlite3.Connection) -> _EntitySelection: raise _translate_error(error) from None +def _lexical_catchup_pending_sync( + dependencies: DiscoveryOperationDependencies, + workspace: Workspace, + record_ids: tuple[str, ...], +) -> bool: + """Return whether the lexical projection still owes these records. + + True only when the records are absent from the active lexical generation + *and* a rebuild job is queued or running, i.e. a retry can actually make + progress. A dead-lettered rebuild, a permanently unavailable index, or a + record that simply does not match the entity name is not retryable. + """ + + if not record_ids: + return False + + def reader(connection: sqlite3.Connection) -> bool: + active = connection.execute( + "SELECT generation FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND status='active'", + (workspace.workspace_id,), + ).fetchone() + indexed = 0 + if active is not None: + placeholders = ",".join("?" for _ in record_ids) + indexed = int( + connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + f"AND projection_generation=? AND record_id IN ({placeholders})", + (workspace.workspace_id, int(active[0]), *record_ids), + ).fetchone()[0] + ) + if indexed >= len(record_ids): + return False + return ( + connection.execute( + "SELECT 1 FROM background_jobs WHERE workspace_id=? " + "AND job_type='retrieval.projection_rebuild' " + "AND status IN ('queued','running') LIMIT 1", + (workspace.workspace_id,), + ).fetchone() + is not None + ) + + try: + return bool(_read_snapshot(dependencies, workspace, reader)) + except Exception: + return False + + def _generation_is_current_sync( dependencies: DiscoveryOperationDependencies, workspace: Workspace, @@ -2074,7 +2124,21 @@ async def _memory_recall_entity( raise DiscoveryOperationError("CAPABILITY_DEGRADED") indexed[record_id] = item.record if set(indexed) != set(selection.record_ids): - raise DiscoveryOperationError("CAPABILITY_DEGRADED") + missing = tuple( + record_id + for record_id in selection.record_ids + if record_id not in indexed + ) + # Only a projection that has genuinely not caught up yet is worth + # retrying. A permanently unavailable or dead-lettered lexical + # index stays a terminal CAPABILITY_DEGRADED. + pending = await _run_blocking( + dependencies, + lambda: _lexical_catchup_pending_sync(dependencies, workspace, missing), + ) + raise DiscoveryOperationError( + "DATABASE_IN_USE" if pending else "CAPABILITY_DEGRADED" + ) await _run_blocking( dependencies, lambda: _generation_is_current_sync( diff --git a/daem0nmcp/api/v7/errors.py b/daem0nmcp/api/v7/errors.py index c42fe14..f240673 100644 --- a/daem0nmcp/api/v7/errors.py +++ b/daem0nmcp/api/v7/errors.py @@ -6,9 +6,12 @@ from __future__ import annotations +import sqlite3 from enum import Enum from types import MappingProxyType +from ...bounded_workers import BoundedWorkerBusyError + class ErrorCode(str, Enum): """Reviewed business-error codes returned inside ``ApiResponse``.""" @@ -63,11 +66,44 @@ def is_stable_error_code(value: object) -> bool: return isinstance(value, str) and value in STABLE_ERROR_CODE_SET +# Clients back off this long before retrying DATABASE_IN_USE. +DATABASE_IN_USE_RETRY_AFTER_MS = 250 + + +def is_database_busy(error: BaseException | None) -> bool: + """Return whether *error* is transient contention a caller may retry. + + A full worker pool and a SQLite lock or busy timeout both clear once the + competing writer finishes, so they map to retryable ``DATABASE_IN_USE``. + The ``code`` check keeps an already-classified repository error (which is + not a ``RuntimeServiceError``) busy through the briefing/preflight wrappers. + """ + + if isinstance(error, BoundedWorkerBusyError): + return True + if getattr(error, "code", None) == ErrorCode.DATABASE_IN_USE.value: + return True + if not isinstance(error, sqlite3.OperationalError): + return False + code = getattr(error, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in {5, 6} # SQLITE_BUSY, SQLITE_LOCKED + # sqlite_errorcode is unavailable on Python 3.10; match SQLite's fixed + # lock diagnostics only, never arbitrary detail text. + return str(error).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + __all__ = [ + "DATABASE_IN_USE_RETRY_AFTER_MS", "ERROR_CODE_REGISTRY", "INTERNAL_ERROR_MESSAGE", "STABLE_ERROR_CODES", "STABLE_ERROR_CODE_SET", "ErrorCode", + "is_database_busy", "is_stable_error_code", ] diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index ff1e6b4..d9f26ad 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -917,6 +917,8 @@ def _projection_sync( raise CoreOperationError("CAPABILITY_DISABLED") from exc if exc.code in {"LEXICAL_UNAVAILABLE", "FTS5_UNAVAILABLE"}: raise CoreOperationError("LEXICAL_UNAVAILABLE") from exc + if exc.code == "DATABASE_IN_USE": + raise CoreOperationError("DATABASE_IN_USE") from exc raise CoreOperationError("CAPABILITY_DEGRADED") from exc except CoreOperationError: raise diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 1a873ef..41e9aae 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -3,13 +3,17 @@ from __future__ import annotations import inspect +import logging import os +import re from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType from typing import Any, Protocol +from pydantic import ValidationError + from ...covenant import ( ArgumentNormalizationError, CovenantGate, @@ -19,8 +23,14 @@ from ...event_store import AppendedEvent, EventStreamConflict from ...retrieval import RetrievalQuery from ...workspace import Workspace -from .errors import ErrorCode -from .models import ApiResponse, CapabilityState, RecordSummary, RetrievalData +from .errors import DATABASE_IN_USE_RETRY_AFTER_MS, ErrorCode +from .models import ( + ApiResponse, + ApiWarning, + CapabilityState, + RecordSummary, + RetrievalData, +) from .responses import ResponseContext, ResponseFactory from .tasks import ( durable_task_execution_var, @@ -77,6 +87,39 @@ def __init__(self, code: str) -> None: super().__init__(code) +_FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") + + +def _token_not_issued_warning(error: ArgumentNormalizationError) -> ApiWarning: + """Say why a description-only preflight carries no token. + + Supplied values are never echoed: only pydantic error types and the + locations of the failing arguments. A location part is echoed only when + it is an int index or identifier-shaped (which may include a key the + caller sent); anything else, such as a path-like key, becomes ``?``. + """ + + problems: list[str] = [] + cause = error.__cause__ + if isinstance(cause, ValidationError): + for item in cause.errors(): + location = ".".join( + str(part) + if isinstance(part, int) or _FIELD_NAME.match(str(part)) + else "?" + for part in item["loc"] + ) + problems.append(f"{location or 'target_arguments'} ({item['type']})") + detail = "; ".join(problems)[:360] or "they do not match the target tool schema" + return ApiWarning( + code="PREFLIGHT_TOKEN_NOT_ISSUED", + message=( + f"Target arguments did not validate: {detail}. " + "No preflight token issued; fix them and call memory_preflight again." + ), + ) + + _EXPECTED_SERVICE_ERRORS = MappingProxyType( { "NOT_FOUND": ( @@ -159,7 +202,23 @@ def _expected_service_failure( if mapped is None: return None code, message, retryable = mapped - return response.failure(code, message, retryable=retryable) + if error_code == "ACTIVE_V7_UNAVAILABLE": + # Opaque on the wire like INTERNAL_ERROR, so keep the cause findable. + logging.getLogger(__name__).warning( + "v7 workspace unavailable correlation_id=%s", + response.request_id, + exc_info=error, + ) + return response.failure( + code, + message, + retryable=retryable, + retry_after_ms=( + DATABASE_IN_USE_RETRY_AFTER_MS + if code is ErrorCode.DATABASE_IN_USE + else None + ), + ) def _utc_now() -> datetime: @@ -474,13 +533,14 @@ async def memory_preflight( **request.target_arguments, } target_is_complete = True + draft_warning: ApiWarning | None = None try: normalized_arguments = self._dependencies.argument_normalizer( request.target_tool, target_call_arguments, scope.canonical_workspace, ) - except ArgumentNormalizationError: + except ArgumentNormalizationError as normalization_error: if request.description is None: return response.failure( ErrorCode.INVALID_ARGUMENT, @@ -491,6 +551,7 @@ async def memory_preflight( # can never become capability material until full normalization # succeeds. target_is_complete = False + draft_warning = _token_not_issued_warning(normalization_error) normalized_arguments = dict(request.target_arguments) try: guidance_value = self._dependencies.preflight_service.guidance( @@ -513,6 +574,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=None, ), + warnings=([draft_warning] if draft_warning is not None else ()), ) token = self._dependencies.covenant_gate.issue_preflight( scope, @@ -760,6 +822,14 @@ async def memory_store( if failure is not None: return failure assert scope is not None + preflight_remedy = { + "workspace_id": request.workspace_id, + "target_tool": "memory_store", + "target_arguments": request.model_dump( + mode="json", + exclude={"workspace_id", "preflight_token"}, + ), + } violation = self._dependencies.covenant_gate.authorize( "memory_store", request.model_dump(), @@ -786,14 +856,7 @@ async def memory_store( ErrorCode.TOKEN_LEGACY_UNSUPPORTED, }: remedy_tool = "memory_preflight" - remedy_arguments = { - "workspace_id": request.workspace_id, - "target_tool": "memory_store", - "target_arguments": request.model_dump( - mode="json", - exclude={"workspace_id", "preflight_token"}, - ), - } + remedy_arguments = preflight_remedy return response.failure( stable_code, "The preflight capability was rejected.", @@ -842,6 +905,18 @@ async def memory_store( retryable=True, ) except Exception as exc: + if getattr(exc, "code", None) == ErrorCode.DATABASE_IN_USE.value: + # authorize() already spent the token, so a retry needs a new + # one; the idempotency key keeps the retry replay-safe. + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use. Request a new " + "preflight token, then retry with the same idempotency_key.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool="memory_preflight", + remedy_arguments=preflight_remedy, + ) return _expected_service_failure(response, exc) or response.internal_error( exc ) diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 498cd4a..c114041 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -605,6 +605,7 @@ async def read( failure_limit=20, rule_limit=50, active_context_limit=1, + include_git_changes=False, ) candidate_records = [ *_public_items(snapshot.failures, RecordSummary), diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index 30f5ebc..ccee58a 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -30,6 +30,7 @@ from ...schema_version import CURRENT_SCHEMA_VERSION from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path +from .errors import is_database_busy from .models import RecordSummary from .public_ids import PublicObjectIdRepository from .resources import ( @@ -209,8 +210,9 @@ class ResourceRepositoryError(RuntimeError): code = "RESOURCE_REPOSITORY_UNAVAILABLE" - def __init__(self) -> None: - super().__init__(self.code) + def __init__(self, code: str = "RESOURCE_REPOSITORY_UNAVAILABLE") -> None: + self.code = code + super().__init__(code) ActiveDatabaseResolver = Callable[[Workspace], ResolvedActiveDatabase] @@ -493,8 +495,13 @@ async def read_briefing_snapshot( failure_limit: int, rule_limit: int = 50, active_context_limit: int = 50, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: - """Read all briefing resources without a pointer-generation gap.""" + """Read all briefing resources without a pointer-generation gap. + + Git changes cost two ``git`` subprocesses per read (plus a system-wide + thread walk on Windows); callers that never show them skip them. + """ if not isinstance(workspace, Workspace): raise ValueError("workspace must be a registered Workspace") @@ -535,6 +542,7 @@ async def read_briefing_snapshot( failure_request=failure_request, rule_request=rule_request, active_request=active_request, + include_git_changes=include_git_changes is True, ) ) @@ -546,6 +554,7 @@ def _read_briefing_snapshot_sync( failure_request: ResourceReadRequest | None, rule_request: ResourceReadRequest, active_request: ResourceReadRequest, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: connection, storage_lock = self._open_connection(workspace) try: @@ -617,7 +626,9 @@ def _read_briefing_snapshot_sync( rules=rules, active_context=active_context, decisions=decisions, - git_changes=self._read_git_changes_sync(workspace), + git_changes=( + self._read_git_changes_sync(workspace) if include_git_changes else [] + ), projection_freshness=projection_freshness, workspace_statistics=workspace_statistics, stale_projection_count=stale_projection_count, @@ -923,6 +934,7 @@ def _workspace_statistics_sync( async def _run(self, operation: Callable[[], T]) -> T: worker = asyncio.create_task(self._worker_pool.run(operation)) cancellation: asyncio.CancelledError | None = None + timed_out = False try: return cast( T, @@ -932,7 +944,7 @@ async def _run(self, operation: Callable[[], T]) -> T: ), ) except asyncio.TimeoutError: - pass + timed_out = True except asyncio.CancelledError as exc: cancellation = exc except Exception: @@ -952,12 +964,16 @@ async def _run(self, operation: Callable[[], T]) -> T: continue except Exception: break - if worker.done(): - with suppress(Exception): - worker.result() + failure: BaseException | None = None + if worker.done() and not worker.cancelled(): + failure = worker.exception() if cancellation is not None: raise cancellation - raise ResourceRepositoryError() + # A read that overran its deadline but did not itself fail was slowed + # by contention (lock waits, a saturated host), so it is retryable. + if is_database_busy(failure) or (timed_out and failure is None): + raise ResourceRepositoryError("DATABASE_IN_USE") from failure + raise ResourceRepositoryError() from failure def _open_connection( self, diff --git a/daem0nmcp/api/v7/responses.py b/daem0nmcp/api/v7/responses.py index fa51258..b913108 100644 --- a/daem0nmcp/api/v7/responses.py +++ b/daem0nmcp/api/v7/responses.py @@ -2,12 +2,18 @@ from __future__ import annotations +import logging +import re import secrets +import sys +import traceback from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, TypeVar +from pydantic import ValidationError + from .errors import INTERNAL_ERROR_MESSAGE, ErrorCode from .models import ( ApiError, @@ -22,6 +28,52 @@ T = TypeVar("T") +_LOGGER = logging.getLogger(__name__) +_LOCATION_PART = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") + + +def _exception_chain(error: BaseException) -> list[BaseException]: + """Return *error* and its causes, root cause first, as tracebacks do.""" + + chain: list[BaseException] = [] + link: BaseException | None = error + while link is not None and all(link is not seen for seen in chain): + chain.append(link) + link = link.__cause__ or ( + None if link.__suppress_context__ else link.__context__ + ) + return chain[::-1] + + +def _without_input_values(chain: list[BaseException]) -> str: + """Render a traceback whose pydantic errors omit the offending values. + + ``str(ValidationError)`` quotes a slice of each invalid input, which for + service output can be stored memory content or a token. + """ + + rendered = [] + for link in chain: + if isinstance(link, ValidationError): + detail = "; ".join( + ".".join( + str(part) + if isinstance(part, int) or _LOCATION_PART.match(str(part)) + else "?" + for part in item["loc"] + ) + + f" ({item['type']})" + for item in link.errors(include_url=False) + ) + else: + detail = str(link) + rendered.append( + "Traceback (most recent call last):\n" + + "".join(traceback.format_tb(link.__traceback__)) + + f"{type(link).__module__}.{type(link).__qualname__}: {detail}" + ) + return "\n\nThe above exception led to:\n\n".join(rendered) + def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -118,9 +170,28 @@ def failure( ) def internal_error(self, error: BaseException | None = None) -> ApiResponse[Any]: - """Return the one deliberately opaque internal failure envelope.""" - - del error + """Return the one deliberately opaque internal failure envelope. + + The cause is logged (stderr by default, never the MCP wire) under the + correlation ID the caller receives, so the owner can find it. Logs + may still hold short exception messages such as local paths; redact + them before sharing. Pydantic input values are never logged. + """ + + cause = error if error is not None else sys.exc_info()[1] + chain = [] if cause is None else _exception_chain(cause) + if any(isinstance(link, ValidationError) for link in chain): + _LOGGER.error( + "v7 internal error correlation_id=%s\n%s", + self.request_id, + _without_input_values(chain), + ) + else: + _LOGGER.error( + "v7 internal error correlation_id=%s", + self.request_id, + exc_info=cause, + ) return self.failure( ErrorCode.INTERNAL_ERROR, INTERNAL_ERROR_MESSAGE, diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index 09d08c3..a9b1063 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -48,6 +48,7 @@ normalize_resolved_path, resolve_derived_path, ) +from .errors import is_database_busy from .federated_retrieval import ( FederatedCandidate, FederatedRetrievalError, @@ -124,8 +125,8 @@ def _validated_workspace(workspace: Workspace) -> Workspace: try: canonical = normalize_resolved_path(workspace.root.resolve(strict=True)) registered = WorkspaceRegistry([canonical], default_root=canonical).default - except (OSError, RuntimeError, TypeError, ValueError): - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except (OSError, RuntimeError, TypeError, ValueError) as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if ( registered.workspace_id != workspace.workspace_id or os.path.normcase(str(registered.root)) != os.path.normcase(str(canonical)) @@ -146,8 +147,8 @@ def resolve_workspace_storage(workspace: Workspace) -> Path: ".daem0nmcp", "storage", ) - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc if not storage.is_dir() or storage.is_symlink(): raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") return storage @@ -173,9 +174,13 @@ def locked_current( except RuntimeServiceError: lock.release() raise - except Exception: + except Exception as exc: lock.release() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + # DatabaseInUseError: another process holds the storage lock + # exclusively (fresh bootstrap, migration) - transient, retryable. + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc try: yield active finally: @@ -243,10 +248,12 @@ def _open_database(path: Path, *, writable: bool) -> sqlite3.Connection: with suppress(NameError, sqlite3.Error): connection.close() raise - except Exception: + except Exception as exc: with suppress(NameError, sqlite3.Error): connection.close() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc def _inspect_database_health( @@ -298,8 +305,8 @@ def _inspect_database_health( # structurally incomplete storage. Full-page corruption certification # remains the explicit offline verify-v7 operation. return schema_version, tables - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc finally: if connection is not None: with suppress(sqlite3.Error): @@ -314,8 +321,8 @@ def _datetime_us(value: datetime) -> int: epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) delta = utc - epoch result = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds - except (OverflowError, ValueError): - raise RuntimeServiceError("INVALID_TIMESTAMP") from None + except (OverflowError, ValueError) as exc: + raise RuntimeServiceError("INVALID_TIMESTAMP") from exc if not -(2**63) <= result <= 2**63 - 1: raise RuntimeServiceError("INVALID_TIMESTAMP") return result @@ -326,8 +333,8 @@ def _datetime_from_us(value: object) -> datetime: raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") try: return datetime.fromtimestamp(value / 1_000_000, timezone.utc) - except (OverflowError, OSError, ValueError): - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except (OverflowError, OSError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: @@ -343,8 +350,8 @@ def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: def _parse_json(value: object, expected_type: type, code: str) -> Any: try: parsed = json.loads(str(value)) - except (TypeError, ValueError, RecursionError): - raise RuntimeServiceError(code) from None + except (TypeError, ValueError, RecursionError) as exc: + raise RuntimeServiceError(code) from exc if not isinstance(parsed, expected_type): raise RuntimeServiceError(code) return parsed @@ -363,8 +370,8 @@ def _event_receipt(row: sqlite3.Row) -> AppendedEvent: else str(row["previous_event_hash"]) ), ) - except (KeyError, TypeError, ValueError): - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc def _idempotency_correlation( @@ -406,8 +413,8 @@ def _existing_idempotent_event( payload = _parse_json(row["payload_json"], dict, "MEMORY_EVENT_INTEGRITY_FAILED") try: canonical = canonical_json_bytes(payload).decode("utf-8") - except Exception: - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc if canonical != str(row["payload_json"]) or sha256_json(payload) != str( row["payload_hash"] ): @@ -473,8 +480,8 @@ def _record_summary( ) except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_state(row: sqlite3.Row) -> dict[str, Any]: @@ -564,6 +571,8 @@ async def _run_mutation( worker = asyncio.create_task(self._workers.run(lambda: operation(cancelled))) try: return await asyncio.shield(worker) + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("DATABASE_IN_USE") from exc except asyncio.CancelledError as cancellation: cancelled.set() try: @@ -577,8 +586,8 @@ async def _run_mutation( def _now_us(self) -> int: try: value = self._clock() - except Exception: - raise RuntimeServiceError("CLOCK_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("CLOCK_UNAVAILABLE") from exc return _datetime_us(value) def _schedule_after_commit(self, path: Path, changed: bool) -> None: @@ -739,10 +748,12 @@ def _store_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_STORE_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_STORE_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -859,10 +870,12 @@ def _record_outcome_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -928,8 +941,8 @@ def _retrieval_config_fingerprint( "capabilities": dict(sorted(capability_statuses.items())), } ) - except (TypeError, ValueError): - raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from None + except (TypeError, ValueError) as exc: + raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from exc @dataclass(slots=True) @@ -1063,8 +1076,8 @@ async def _run_snapshot(self, operation: Callable[[], Any]) -> Any: worker = asyncio.create_task(self._workers.run(operation)) try: return await asyncio.shield(worker) - except BoundedWorkerBusyError: - raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from None + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from exc except asyncio.CancelledError: with suppress(asyncio.CancelledError, Exception): await await_task_terminal(worker) @@ -1111,8 +1124,8 @@ async def retrieve( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc return hydrated async def _retrieve_source( @@ -1144,8 +1157,8 @@ def _resolve_federated_workspace(self, workspace_id: str) -> Workspace: try: workspace = resolve(workspace_id) validated = _validated_workspace(workspace) - except Exception: - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except Exception as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if validated.workspace_id != workspace_id: raise RuntimeServiceError("INVALID_WORKSPACE") return validated @@ -1233,10 +1246,10 @@ async def retrieve_one( hydrated = compose_federated_results(dict(pairs), query) finally: guard.release() - except asyncio.TimeoutError: - raise RuntimeServiceError("DEADLINE_EXCEEDED") from None + except asyncio.TimeoutError as exc: + raise RuntimeServiceError("DEADLINE_EXCEEDED") from exc except FederatedRetrievalError as exc: - raise RuntimeServiceError(exc.code) from None + raise RuntimeServiceError(exc.code) from exc except asyncio.CancelledError: raise except RuntimeServiceError: @@ -1247,8 +1260,8 @@ async def retrieve_one( "COMMUNION_REQUIRED", "UNAUTHORIZED_WORKSPACE", }: - raise RuntimeServiceError(authorization_code) from None - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + raise RuntimeServiceError(authorization_code) from exc + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc # `hydrated` is immutable and there is no await between releasing the # read guard above and committing this response to the caller. return hydrated @@ -1348,10 +1361,10 @@ def _hydrate_candidates( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1403,8 +1416,8 @@ def _hydrate( dropped=0, ), ) - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc if result.context is None: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") @@ -1516,10 +1529,10 @@ def _hydrate( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1540,8 +1553,8 @@ def _public_ref(ref: EvidenceRef, origin_workspace_id: str) -> PublicEvidenceRef relation_path=list(ref.relation_path), provider=provider, ) - except Exception: - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc class BriefingReader(Protocol): @@ -1593,8 +1606,10 @@ async def assemble( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("BRIEFING_FAILED") from None + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("BRIEFING_FAILED") from exc if data.workspace_id != workspace.workspace_id: raise RuntimeServiceError("BRIEFING_FAILED") return data @@ -1649,8 +1664,10 @@ async def guidance( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("PREFLIGHT_FAILED") from None + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("PREFLIGHT_FAILED") from exc return data diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index 08aae92..4bbd03d 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -378,6 +378,19 @@ def main(): "--workspace-id", required=True, help="Registered opaque workspace ID" ) + compact_projections_parser = subparsers.add_parser( + "compact-projections", + help="Offline: drop every superseded projection generation and VACUUM", + ) + compact_projections_parser.add_argument( + "--workspace-id", required=True, help="Registered opaque workspace ID" + ) + compact_projections_parser.add_argument( + "--no-vacuum", + action="store_true", + help="Drop the superseded generations without reclaiming the file", + ) + rebuild_projection_parser = subparsers.add_parser( "rebuild-projection", help="Build and activate one v7 retrieval projection" ) @@ -780,11 +793,16 @@ def main(): print(f"Runs: {len(payload['runs'])}") sys.exit(exit_code) - if args.command in {"projection-status", "rebuild-projection"}: + if args.command in { + "compact-projections", + "projection-status", + "rebuild-projection", + }: import sqlite3 from .retrieval.operations import ( ProjectionOperationError, + compact_projections, projection_status, rebuild_projection, ) @@ -812,7 +830,12 @@ def main(): update={"project_root": str(workspace.root)} ) storage_path = Path(workspace_settings.get_storage_path()) - with DatabaseFileLock(storage_path, "shared"): + # Compaction drops generations a running server could be reading, + # so it takes the offline exclusive lock. + lock_mode = ( + "exclusive" if args.command == "compact-projections" else "shared" + ) + with DatabaseFileLock(storage_path, lock_mode): active = resolve_active_database(storage_path) if active.format_version != 7: raise ProjectionOperationError("FORMAT_7_REQUIRED") @@ -823,6 +846,12 @@ def main(): connection.execute("PRAGMA foreign_keys=ON") if args.command == "projection-status": payload = projection_status(connection, workspace.workspace_id) + elif args.command == "compact-projections": + payload = compact_projections( + connection, + workspace.workspace_id, + vacuum=not args.no_vacuum, + ) else: builders = create_projection_builders( connection, diff --git a/daem0nmcp/event_store.py b/daem0nmcp/event_store.py index bfea017..e7328b6 100644 --- a/daem0nmcp/event_store.py +++ b/daem0nmcp/event_store.py @@ -11,6 +11,7 @@ import json import math import re +import sqlite3 import unicodedata from collections.abc import Iterator, Mapping from dataclasses import dataclass @@ -536,7 +537,10 @@ def _append(self, command: EventCommand, payload: dict[str, Any]) -> AppendedEve """, values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # Only a uniqueness/constraint collision is a stream conflict; I/O + # and lock errors propagate so callers do not advise a retry of a + # conflict that never happened. raise EventStreamConflict( "event identity or stream version is occupied" ) from exc @@ -1933,7 +1937,8 @@ def _append( "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # As for memory events: only a constraint collision is a conflict. raise EventStreamConflict( "governance event identity or stream version is occupied" ) from exc diff --git a/daem0nmcp/retrieval/jobs.py b/daem0nmcp/retrieval/jobs.py index 4376bbd..f9f7a60 100644 --- a/daem0nmcp/retrieval/jobs.py +++ b/daem0nmcp/retrieval/jobs.py @@ -26,6 +26,26 @@ _HEARTBEAT_SHUTDOWN_GRACE_SECONDS = 1.5 +def is_lock_contention(exc: BaseException) -> bool: + """Return whether *exc* is SQLite lock contention a caller may retry.""" + + if not isinstance(exc, sqlite3.OperationalError): + return False + code = getattr(exc, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + } + # sqlite_errorcode is unavailable on supported Python 3.10 builds. + # Match only SQLite's fixed lock diagnostics, never arbitrary detail. + return str(exc).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + class ProjectionJobError(RuntimeError): """Owned, sanitized projection-job failure.""" @@ -250,7 +270,7 @@ def _heartbeat_loop( break except sqlite3.OperationalError as exc: connection.rollback() - if not self._is_lock_contention(exc): + if not is_lock_contention(exc): raise if stop.is_set(): if stop_seen_at is None: @@ -554,22 +574,6 @@ def _now(self) -> int: raise ProjectionJobError("PROJECTION_JOB_CLOCK_INVALID") return value - @staticmethod - def _is_lock_contention(exc: sqlite3.OperationalError) -> bool: - code = getattr(exc, "sqlite_errorcode", None) - if isinstance(code, int): - return code & 0xFF in { - sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_LOCKED, - } - # sqlite_errorcode is unavailable on supported Python 3.10 builds. - # Match only SQLite's fixed lock diagnostics, never arbitrary detail. - return str(exc).casefold() in { - "database is locked", - "database table is locked", - "database schema is locked", - } - def create_projection_job_runner( connection: sqlite3.Connection, @@ -593,4 +597,5 @@ def create_projection_job_runner( "ProjectionJobRun", "ProjectionJobRunner", "create_projection_job_runner", + "is_lock_contention", ] diff --git a/daem0nmcp/retrieval/lexical_config.py b/daem0nmcp/retrieval/lexical_config.py index d23c243..e87797f 100644 --- a/daem0nmcp/retrieval/lexical_config.py +++ b/daem0nmcp/retrieval/lexical_config.py @@ -18,6 +18,14 @@ "tokenizer": LEXICAL_TOKENIZER, } +# Per-generation row tables of the rebuildable local projections. Temporal +# keeps no rows of its own; graph generations are owned by discovery tables. +GENERATION_TABLES = { + "lexical": "retrieval_documents", + "procedure": "record_procedures", + "outcome": "record_outcome_view", +} + _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _FTS_PARTITION = re.compile(r"^retrieval_fts_[0-9a-f]{24}_g[1-9][0-9]*$") @@ -48,6 +56,7 @@ def lexical_fts_table_name(workspace_id: str, generation: int) -> str: __all__ = [ + "GENERATION_TABLES", "LEXICAL_BM25_WEIGHTS", "LEXICAL_BUILD_CONFIG", "LEXICAL_TOKENIZER", diff --git a/daem0nmcp/retrieval/operations.py b/daem0nmcp/retrieval/operations.py index ab48297..08935e2 100644 --- a/daem0nmcp/retrieval/operations.py +++ b/daem0nmcp/retrieval/operations.py @@ -9,7 +9,12 @@ from dataclasses import asdict from typing import Any -from .projections import LexicalProjectionBuilder, ProjectionBuildError +from .projections import ( + COLLECTABLE_PROJECTIONS, + LexicalProjectionBuilder, + ProjectionBuildError, + collect_superseded_generations, +) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _PROJECTIONS = frozenset( @@ -157,8 +162,81 @@ def projection_status( } +def compact_projections( + connection: sqlite3.Connection, + workspace_id: str, + *, + vacuum: bool = True, +) -> dict[str, Any]: + """Drain every superseded generation offline, then reclaim the file. + + Online activations collect at most five generations each, so a database + that grew before the garbage collector existed needs about a hundred + writes to drain — and a ``VACUUM`` before that only rewrites the dead + generations. This runs the same collector in a loop until it reports + nothing left, so the following ``VACUUM`` actually reclaims the space. + The caller owns the offline exclusive lock; no activation may run + concurrently. + """ + + if not isinstance(connection, sqlite3.Connection): + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + _validate_workspace(workspace_id) + if connection.in_transaction: + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + collected: dict[str, int] = {} + try: + before = _generation_counts(connection, workspace_id) + for projection in sorted(COLLECTABLE_PROJECTIONS): + removed = 0 + while True: + passed = collect_superseded_generations( + connection, workspace_id, projection + ) + if passed == 0: + break + removed += passed + collected[projection] = removed + if vacuum: + connection.execute("VACUUM") + after = _generation_counts(connection, workspace_id) + except sqlite3.Error as exc: + raise ProjectionOperationError("PROJECTION_COMPACTION_FAILED") from exc + return { + "collected": collected, + "superseded_manifests_after": after[0], + "superseded_manifests_before": before[0], + "vacuumed": bool(vacuum), + "workspace_id": workspace_id, + } + + +def _generation_counts( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[int, int]: + placeholders = ",".join("?" for _ in COLLECTABLE_PROJECTIONS) + projections = sorted(COLLECTABLE_PROJECTIONS) + superseded = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders}) AND status<>'active'", + (workspace_id, *projections), + ).fetchone()[0] + ) + total = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders})", + (workspace_id, *projections), + ).fetchone()[0] + ) + return superseded, total + + __all__ = [ "ProjectionOperationError", + "compact_projections", "projection_status", "rebuild_projection", ] diff --git a/daem0nmcp/retrieval/projections.py b/daem0nmcp/retrieval/projections.py index 4114d29..2548719 100644 --- a/daem0nmcp/retrieval/projections.py +++ b/daem0nmcp/retrieval/projections.py @@ -4,6 +4,7 @@ import hashlib import json +import logging import re import sqlite3 import time @@ -11,14 +12,22 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention from .lexical_config import ( + GENERATION_TABLES, LEXICAL_TOKENIZER, lexical_build_config_hash, lexical_fts_table_name, ) +from .specialized_contract import procedure_fts_table_name + +logger = logging.getLogger(__name__) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _LEXICAL_BUILDER_VERSION = "retrieval-lexical-1" +# Graph is excluded: the discovery tables reference its generations. +COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) +_GC_BUSY_TIMEOUT_MS = 250 class ProjectionBuildError(RuntimeError): @@ -284,11 +293,17 @@ def rebuild( self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") if isinstance(exc, ProjectionBuildError): raise + if is_lock_contention(exc): + raise ProjectionBuildError( + "DATABASE_IN_USE", "lexical projection build was contended" + ) from exc if isinstance(exc, sqlite3.Error): raise ProjectionBuildError( "LEXICAL_UNAVAILABLE", "FTS5 projection build is unavailable" ) from exc raise + if owns_transaction: + collect_superseded_generations(self.connection, workspace_id, "lexical") return result def apply_active_delta(self, workspace_id: str, record_id: str) -> bool: @@ -748,8 +763,152 @@ def _clock_value(self) -> int: return value +_VICTIM_QUERY = """ +SELECT manifest_id, generation FROM projection_manifests +WHERE workspace_id=? AND projection_name=? AND typeof(generation)='integer' + AND ( + ( + status IN ('ready','failed') + AND manifest_id IS NOT ( + SELECT manifest_id FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='ready' + AND activated_at_us IS NOT NULL + ORDER BY generation DESC LIMIT 1 + ) + ) + OR ( + status='rebuild_required' + AND generation < COALESCE(( + SELECT generation FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='active' + AND typeof(generation)='integer' + ), -1) + ) + ) +ORDER BY generation LIMIT ? +""" + + +def _drop_generation( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + manifest_id: str, + generation: int, +) -> None: + fts_table = ( + lexical_fts_table_name(workspace_id, generation) + if projection == "lexical" + else procedure_fts_table_name(workspace_id, generation) + if projection == "procedure" + else None + ) + if fts_table is not None: + connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') + generation_table = GENERATION_TABLES.get(projection) + if generation_table is not None: + connection.execute( + f'DELETE FROM "{generation_table}" ' + "WHERE workspace_id=? AND projection_generation=?", + (workspace_id, generation), + ) + connection.execute( + "DELETE FROM projection_manifests WHERE manifest_id=?", + (manifest_id,), + ) + + +def collect_superseded_generations( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + limit: int = 5, +) -> int: + """Drop up to *limit* superseded generations of one local projection. + + Runs after an activation commits, in its own short transaction. Victims + are chosen by manifest status: ``ready`` and ``failed`` rows, plus + ``rebuild_required`` rows below the active generation (readers consult + those only while no active manifest exists). It never touches the active + or a ``building`` generation, nor the most recently deactivated one (the + highest activated ``ready`` generation), because a reader that resolved + the old manifest just before the switch may still be querying it. + + Best effort: one damaged generation is skipped rather than wedging the + rest, contention ends the pass, and nothing raises — the next activation + retries. Returns the number of generations removed. + """ + + if ( + projection not in COLLECTABLE_PROJECTIONS + or _WORKSPACE_ID.fullmatch(workspace_id) is None + or connection.in_transaction + ): + return 0 + collected = 0 + try: + previous_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0]) + connection.execute(f"PRAGMA busy_timeout={_GC_BUSY_TIMEOUT_MS}") + try: + connection.execute("BEGIN IMMEDIATE") + try: + victims = connection.execute( + _VICTIM_QUERY, + ( + workspace_id, + projection, + workspace_id, + projection, + workspace_id, + projection, + limit, + ), + ).fetchall() + for manifest_id, generation in victims: + connection.execute("SAVEPOINT retrieval_generation_gc") + try: + _drop_generation( + connection, + workspace_id, + projection, + str(manifest_id), + int(generation), + ) + except Exception as exc: + connection.execute( + "ROLLBACK TO SAVEPOINT retrieval_generation_gc" + ) + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + if is_lock_contention(exc): + raise + # A damaged generation must not wedge every later + # one; skip past it and keep collecting. + logger.warning( + "projection generation could not be collected", + exc_info=True, + ) + continue + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + collected += 1 + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.execute(f"PRAGMA busy_timeout={previous_timeout}") + except Exception as exc: + if is_lock_contention(exc): + logger.debug("projection generation GC skipped: database busy") + else: + logger.warning("projection generation GC failed", exc_info=True) + return 0 + return collected + + __all__ = [ + "COLLECTABLE_PROJECTIONS", "LexicalProjectionBuilder", "ProjectionBuildError", "ProjectionBuildResult", + "collect_superseded_generations", ] diff --git a/daem0nmcp/retrieval/specialized_projection.py b/daem0nmcp/retrieval/specialized_projection.py index a5d8043..1282ee5 100644 --- a/daem0nmcp/retrieval/specialized_projection.py +++ b/daem0nmcp/retrieval/specialized_projection.py @@ -11,6 +11,8 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention +from .projections import collect_superseded_generations from .specialized_contract import ( SPECIALIZED_BUILDER_VERSION, specialized_projection_contract, @@ -357,17 +359,28 @@ def rebuild( self.connection.commit() else: self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") - return result except Exception as exc: self._rollback(owns_transaction, savepoint) if isinstance(exc, SpecializedProjectionBuildError): raise + if is_lock_contention(exc): + raise SpecializedProjectionBuildError( + "DATABASE_IN_USE", + "specialized projection build was contended", + ) from exc if isinstance(exc, sqlite3.Error): raise SpecializedProjectionBuildError( "PROJECTION_UNAVAILABLE", "specialized projection build is unavailable", ) from exc raise + if owns_transaction: + # Graph is not collectable: the discovery tables reference its + # generations. collect_superseded_generations enforces that. + collect_superseded_generations( + self.connection, workspace_id, projection_name + ) + return result def active_is_current(self, workspace_id: str, projection_name: str) -> bool: """Return whether the active specialized generation is exact.""" diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index be1003b..eed1d3d 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -35,6 +35,7 @@ ) from .migrations import MIGRATIONS from .migrations.v7 import _source_row_hash, inventory_database +from .retrieval.lexical_config import GENERATION_TABLES from .schema_version import CURRENT_SCHEMA_VERSION, REQUIRED_V7_SCHEMA_VERSIONS from .storage_activation import ( ActiveDatabasePointer, @@ -92,11 +93,6 @@ "entities", "communities", } -_GENERATION_TABLES = { - "lexical": "retrieval_documents", - "procedure": "record_procedures", - "outcome": "record_outcome_view", -} class VerificationV7Error(RuntimeError): @@ -664,6 +660,34 @@ def _verify_mappings(connection: sqlite3.Connection, replay: sqlite3.Connection) return count +def _generation_inventory(connection: sqlite3.Connection) -> dict[str, int]: + """Count superseded local generations so a stalled GC is visible.""" + + from .retrieval.projections import COLLECTABLE_PROJECTIONS + + projections = sorted(COLLECTABLE_PROJECTIONS) + placeholders = ",".join("?" for _ in projections) + superseded = { + str(row[0]): int(row[1]) + for row in connection.execute( + "SELECT projection_name,COUNT(*) FROM projection_manifests " + f"WHERE projection_name IN ({placeholders}) AND status<>'active' " + "GROUP BY projection_name", + projections, + ) + } + inventory = {name: superseded.get(name, 0) for name in projections} + inventory["fts_partitions"] = int( + connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*' " + "AND (name GLOB 'retrieval_fts_*_g*' " + "OR name GLOB 'retrieval_procedure_fts_*_g*')" + ).fetchone()[0] + ) + return inventory + + def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: from .retrieval.projections import LexicalProjectionBuilder from .retrieval.specialized_projection import SpecializedProjectionBuilder @@ -679,7 +703,7 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: "source_event_root_hash,details_json,generation,row_count," "cursor_recorded_at_us,cursor_event_id FROM projection_manifests" ).fetchall() - for name, generation_table in _GENERATION_TABLES.items(): + for name, generation_table in GENERATION_TABLES.items(): if connection.execute( f'SELECT 1 FROM "{generation_table}" rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " @@ -934,6 +958,14 @@ def _verify_database( ) except Exception as exc: checks["projection_manifests"] = _check(False, error=type(exc).__name__) + try: + # Superseded generations should stay near zero. A number that + # keeps growing means the activation-time GC is failing. + checks["projection_generations"] = _check( + True, **_generation_inventory(source) + ) + except Exception as exc: + checks["projection_generations"] = _check(False, error=type(exc).__name__) try: invalid = int( source.execute( @@ -1115,7 +1147,7 @@ def _refresh_manifests(candidate: Path, workspace_ids: list[str]) -> None: ) # Remove only orphaned local generations in the candidate. Otherwise a # renamed/deleted manifest can leave an FTS table occupying the next ID. - for name, table in _GENERATION_TABLES.items(): + for name, table in GENERATION_TABLES.items(): connection.execute( f'DELETE FROM "{table}" AS rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " diff --git a/docs/release/v7/inventory.md b/docs/release/v7/inventory.md index db82039..cc683e0 100644 --- a/docs/release/v7/inventory.md +++ b/docs/release/v7/inventory.md @@ -2,7 +2,7 @@ Generated by `python scripts/v7_release_inventory.py --write`. Registration is not acceptance; only an `accepted` requirement with explicit evidence metadata can be accepted. -Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **571**. +Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **572**. ## Release ledger @@ -131,6 +131,7 @@ Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resou | hook | `hooks/daem0n_stop_hook.py` | `hooks/daem0n_stop_hook.py` | Hook hooks/daem0n_stop_hook.py runs with its documented input and redacted failure behavior. — pending evidence | pending | | cli | `briefing` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli briefing parses and exercises its documented result contract. — pending evidence | pending | | cli | `check` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli check parses and exercises its documented result contract. — pending evidence | pending | +| cli | `compact-projections` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract. — pending evidence | pending | | cli | `index` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli index parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-claude-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-claude-hooks parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-hooks parses and exercises its documented result contract. — pending evidence | pending | diff --git a/docs/release/v7/requirements.json b/docs/release/v7/requirements.json index 6346d97..8b452d4 100644 --- a/docs/release/v7/requirements.json +++ b/docs/release/v7/requirements.json @@ -4370,6 +4370,16 @@ "package": "daem0nmcp.cli", "status": "pending" }, + { + "acceptance": "python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract.", + "entry_point": "daem0nmcp.cli:main", + "evidence": [], + "id": "cli.compact-projections", + "kind": "cli", + "owner": "cli", + "package": "daem0nmcp.cli", + "status": "pending" + }, { "acceptance": "python -m daem0nmcp.cli index parses and exercises its documented result contract.", "entry_point": "daem0nmcp.cli:main", @@ -5744,8 +5754,8 @@ "tests/test_legacy_vector_migration_guards.py" ], "observed_surface": { - "atomic_requirements": 571, - "cli_commands": 20, + "atomic_requirements": 572, + "cli_commands": 21, "dependency_profiles": 10, "hooks": 15, "registered_tools": 75, diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index 1f2f795..e31e2f9 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -160,6 +160,7 @@ async def process_client( environment_overrides: dict[str, str] | None = None, http_probe: Callable[[str], Awaitable[None]] | None = None, http_headers: dict[str, str] | None = None, + log_name: str = "stdio-server.log", ): """Launch the installed module in a workspace and clean up on every exit.""" environment = server_environment( @@ -176,7 +177,7 @@ async def process_client( ) body_error: BaseException | None = None try: - with (workspace / "stdio-server.log").open("w", encoding="utf-8") as log: + with (workspace / log_name).open("w", encoding="utf-8") as log: async with ( stdio_client(parameters, errlog=log) as (read, write), ClientSession( diff --git a/tests/api_v7/test_application.py b/tests/api_v7/test_application.py index 023a3d1..d20677d 100644 --- a/tests/api_v7/test_application.py +++ b/tests/api_v7/test_application.py @@ -129,6 +129,59 @@ async def operation(*, workspace: Workspace, request: object) -> object: self.assertEqual(second.error.code, "TOKEN_REPLAYED") self.assertEqual(len(calls), 1) + async def test_database_in_use_is_retryable_and_points_at_a_fresh_preflight( + self, + ) -> None: + from daem0nmcp.api.v7.models import MutationReceipt + + class BusyError(RuntimeError): + code = "DATABASE_IN_USE" + + attempts: list[object] = [] + + async def operation(*, workspace: Workspace, request: object) -> object: + del workspace + attempts.append(request) + if len(attempts) == 1: + raise BusyError("database is locked") + return MutationReceipt( + operation_id="op_archive_retry", + affected_ids=[request.record_id], + event_ids=[], + counts={"changed": 1}, + idempotent_replay=False, + ) + + self.gate.record_briefing(self.scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_id": RECORD_ID, + "archived": True, + } + handler = self._router({"memory_archive_set": operation}).handler( + "memory_archive_set" + ) + token = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + + busy = await handler(**target, preflight_token=token) + self.assertFalse(busy.ok) + self.assertEqual(busy.error.code, "DATABASE_IN_USE") + self.assertTrue(busy.error.retryable) + self.assertEqual(busy.error.retry_after_ms, 250) + self.assertEqual(busy.error.remedy.tool, "memory_preflight") + self.assertEqual( + busy.error.remedy.arguments["target_arguments"], + {"record_id": RECORD_ID, "archived": True}, + ) + # The token was spent before the operation ran: a literal retry is + # refused, and the remedy (a fresh preflight) succeeds. + replayed = await handler(**target, preflight_token=token) + self.assertEqual(replayed.error.code, "TOKEN_REPLAYED") + fresh = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + retried = await handler(**target, preflight_token=fresh) + self.assertTrue(retried.ok) + self.assertEqual(len(attempts), 2) + async def test_protected_durable_execution_requires_exact_worker_context( self, ) -> None: diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index a40b2d8..eab8404 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1162,6 +1162,85 @@ async def retrieve(self, workspace, query, linked_workspace_ids): self.assertEqual(1, len(second.items)) self.assertNotEqual(first.items[0].record_id, second.items[0].record_id) + async def test_memory_recall_entity_retries_only_a_pending_catch_up( + self, + ) -> None: + """Only a projection that still owes the records is worth retrying.""" + from daem0nmcp.api.v7.discovery_operations import DiscoveryOperationError + from daem0nmcp.api.v7.models import ( + ProviderDiagnostic, + RetrievalData, + TokenUsage, + ) + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._activate_discovery() + + class RecallService: + """Returns nothing, as a lexical miss on the entity name does.""" + + async def retrieve(self, workspace, query, linked_workspace_ids): + return RetrievalData( + provider_diagnostics=[ + ProviderDiagnostic( + provider="lexical", + status="unavailable", + manifest_generation=None, + elapsed_ms=0.0, + reason="LEXICAL_UNAVAILABLE", + returned_count=0, + ) + ], + abstained=True, + abstention_reason="NO_EVIDENCE", + token_usage=TokenUsage( + budget=query.token_budget, + requested=0, + selected=0, + rendered=0, + dropped=0, + ), + ) + + operation = self._operations(recall_service=RecallService())[ + "memory_recall_entity" + ] + request = _request( + "memory_recall_entity", + workspace_id=self.workspace.workspace_id, + entity_name="Authentication", + ) + + async def code() -> str: + with self.assertRaises(DiscoveryOperationError) as raised: + await operation(workspace=self.workspace, request=request) + return raised.exception.code + + # The records are not in the lexical projection yet and its rebuild + # job is queued: a retry can make progress. + self.assertEqual("DATABASE_IN_USE", await code()) + + # The rebuild dead-lettered, so nothing will catch up on its own. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute( + "UPDATE background_jobs SET status='dead_letter'," + "finished_at_us=1 WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + + # The projection holds every member, so the miss is genuine even + # though lexical reports itself unavailable and a job is queued. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + LexicalProjectionBuilder(connection).rebuild(self.workspace.workspace_id) + connection.execute( + "UPDATE background_jobs SET status='queued',finished_at_us=NULL " + "WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + async def test_stats_read_the_active_canonical_graph_snapshot(self) -> None: """Counting retained-v6 graph rows would misreport the v7 projection.""" from daem0nmcp.api.v7.tools import KnowledgeGraphStatsData diff --git a/tests/api_v7/test_error_honesty.py b/tests/api_v7/test_error_honesty.py new file mode 100644 index 0000000..d248f3b --- /dev/null +++ b/tests/api_v7/test_error_honesty.py @@ -0,0 +1,279 @@ +"""Transient contention is retryable and every internal error is logged (KD-6).""" + +from __future__ import annotations + +import asyncio +import json +import logging +import sqlite3 +import threading +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client + +from daem0nmcp.api.v7 import runtime_services +from daem0nmcp.api.v7.production import ( + _briefing_reader, + _guidance_reader, + create_v7_server, +) +from daem0nmcp.api.v7.resource_repository import build_sqlite_resource_readers +from daem0nmcp.api.v7.runtime_services import ( + BasicBriefingService, + BasicPreflightService, + RuntimeServiceError, +) +from daem0nmcp.api.v7.tools import SessionBriefInput +from daem0nmcp.bounded_workers import BoundedWorkerPool +from daem0nmcp.config import Settings +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces + +FLOWS = 8 + + +def _server(root: Path): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(root), + workspace_roots=[str(root)], + dream_enabled=False, + ), + environ={}, + ) + + +def _database(root: Path) -> Path: + return resolve_active_database(root / ".daem0nmcp" / "storage").path + + +def _is_retryable_busy(response: dict[str, Any]) -> bool: + error = response["error"] + return ( + error["code"] == "DATABASE_IN_USE" + and error["retryable"] is True + and error["retry_after_ms"] == 250 + ) + + +class _Session: + def __init__(self, client: Client, workspace_id: str) -> None: + self._client = client + self.workspace_id = workspace_id + + async def call(self, tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await self._client.call_tool( + tool, + {"workspace_id": self.workspace_id, **arguments}, + raise_on_error=False, + ) + assert isinstance(result.structured_content, dict), tool + return result.structured_content + + async def preflight_and_store(self, index: int) -> dict[str, Any]: + arguments = { + "record_type": "decision", + "content": f"Concurrent decision number {index}", + "idempotency_key": f"concurrent-store-{index:04d}", + } + preflight = await self.call( + "memory_preflight", + {"target_tool": "memory_store", "target_arguments": arguments}, + ) + if not preflight["ok"]: + return preflight + return await self.call( + "memory_store", + {**arguments, "preflight_token": preflight["data"]["preflight_token"]}, + ) + + +async def test_parallel_stores_are_ok_or_retryable_and_land_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + + async def flow(index: int) -> list[dict[str, Any]]: + observed = [] + deadline = asyncio.get_running_loop().time() + 60 + while True: + response = await session.preflight_and_store(index) + observed.append(response) + if response["ok"]: + return observed + assert _is_retryable_busy(response), response + assert asyncio.get_running_loop().time() < deadline, observed + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + results = await asyncio.gather(*(flow(index) for index in range(FLOWS))) + + for observed in results: + assert observed[-1]["ok"], observed + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) + with closing(sqlite3.connect(_database(tmp_path))) as connection: + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + events = connection.execute( + "SELECT count(*) FROM memory_events WHERE event_type='memory.created'" + ).fetchone() + assert (records[0], events[0]) == (FLOWS, FLOWS) + + +async def test_store_behind_a_held_write_lock_is_retryable_database_in_use(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with closing( + sqlite3.connect(_database(tmp_path), isolation_level=None) + ) as holder: + holder.execute("BEGIN IMMEDIATE") + try: + # Waits out the writer's 5 s SQLite busy timeout. + blocked = await session.preflight_and_store(1) + finally: + holder.execute("ROLLBACK") + assert not blocked["ok"] + assert _is_retryable_busy(blocked), blocked + retried = await session.preflight_and_store(1) + assert retried["ok"], retried + + +async def test_internal_error_is_logged_under_the_returned_correlation_id( + tmp_path, monkeypatch, caplog +): + def explode(*_arguments, **_keywords): + raise RuntimeError("induced-failure-canary") + + # Raised inside the writer transaction, so it reaches the log only through + # the MEMORY_STORE_FAILED exception chain. + monkeypatch.setattr(runtime_services, "_existing_idempotent_event", explode) + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = await session.preflight_and_store(1) + + assert not response["ok"] + error = response["error"] + assert (error["code"], error["retryable"]) == ("INTERNAL_ERROR", False) + assert "induced-failure-canary" not in json.dumps(response) + [record] = [ + record + for record in caplog.records + if record.name == "daem0nmcp.api.v7.responses" + ] + assert error["correlation_id"] in record.getMessage() + assert "induced-failure-canary" in caplog.text + assert "MEMORY_STORE_FAILED" in caplog.text + + +@pytest.mark.parametrize("contention", ["full_pool", "overrun_read"]) +@pytest.mark.parametrize("service", ["memory_preflight", "session_brief"]) +async def test_contended_briefing_snapshot_is_retryable_database_in_use( + tmp_path, service, contention +): + """The CI failure path: guidance/briefing -> read_briefing_snapshot -> _run. + + MCP resource URIs still report an opaque access failure; the repository + code only matters on these two service paths. + """ + [workspace] = await initialize_workspaces((tmp_path,)) + storage = tmp_path / ".daem0nmcp" / "storage" + pool = BoundedWorkerPool(max_workers=1, thread_name_prefix="test-contention") + release = threading.Event() + + def resolve(_workspace): + if contention == "overrun_read": + release.wait(timeout=30) # overrun the 0.05 s deadline, then succeed + return resolve_active_database(storage) + + readers = build_sqlite_resource_readers( + resolve, timeout_seconds=0.05, worker_pool=pool + ) + + async def pool_occupied() -> None: + deadline = asyncio.get_running_loop().time() + 10 + while not pool.in_flight: + assert asyncio.get_running_loop().time() < deadline + await asyncio.sleep(0.01) + + occupant = None + if contention == "full_pool": + occupant = asyncio.create_task(pool.run(lambda: release.wait(timeout=30))) + await pool_occupied() + if service == "memory_preflight": + call = BasicPreflightService(reader=_guidance_reader(readers)).guidance( + workspace, "memory_store", {}, "Plan a decision." + ) + else: + call = BasicBriefingService(reader=_briefing_reader(readers)).assemble( + workspace, SessionBriefInput(workspace_id=workspace.workspace_id) + ) + task = asyncio.ensure_future(call) + try: + if contention == "overrun_read": + await pool_occupied() + await asyncio.sleep(0.2) + release.set() + with pytest.raises(RuntimeServiceError) as caught: + await task + finally: + release.set() + if occupant is not None: + await occupant + pool.shutdown() + assert caught.value.code == "DATABASE_IN_USE" + + +async def test_exclusive_storage_lock_is_database_in_use_not_unavailable(tmp_path): + """A migration/bootstrap holding the storage lock is transient contention.""" + from daem0nmcp.api.v7.runtime_services import WorkspaceStorageResolver + from daem0nmcp.storage_activation import DatabaseFileLock + + [workspace] = await initialize_workspaces((tmp_path,)) + holder = DatabaseFileLock(tmp_path / ".daem0nmcp" / "storage", "exclusive") + holder.acquire() + try: + with ( + pytest.raises(RuntimeServiceError) as caught, + WorkspaceStorageResolver().locked_active(workspace), + ): + pytest.fail("a shared read was admitted under an exclusive lock") + finally: + holder.release() + assert caught.value.code == "DATABASE_IN_USE" + with WorkspaceStorageResolver().locked_active(workspace) as active: + assert active.format_version == 7 + + +def test_internal_error_log_omits_pydantic_input_values(caplog): + from pydantic import BaseModel, ValidationError + + from daem0nmcp.api.v7.responses import ResponseFactory + + class Guidance(BaseModel): + priority: int + + # Built at runtime so the traceback's source lines cannot contain it. + stored_content = "-".join(("secret", "memory", "fragment", "canary")) + try: + try: + Guidance.model_validate({"priority": stored_content}) + except ValidationError as invalid: + raise RuntimeError("PREFLIGHT_FAILED") from invalid + except RuntimeError as error: + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = ResponseFactory().begin(None).internal_error(error) + + assert response.error.correlation_id in caplog.text + assert "priority (int_parsing)" in caplog.text + assert "PREFLIGHT_FAILED" in caplog.text + assert "test_internal_error_log_omits_pydantic_input_values" in caplog.text + assert "secret-memory-fragment-canary" not in caplog.text diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index c72d5a5..3bb5f98 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -1189,6 +1189,12 @@ async def guidance( self.assertTrue(validated.ok) self.assertIsNone(validated.data.preflight_token) self.assertIsNone(validated.data.expires_at) + # An ok:true draft must say why no token was issued (F-070/F-082). + [warning] = validated.meta.warnings + self.assertEqual("PREFLIGHT_TOKEN_NOT_ISSUED", warning.code) + self.assertIn("record_type (missing)", warning.message) + self.assertIn("content (missing)", warning.message) + self.assertIn("No preflight token issued", warning.message) self.assertEqual(len(observed), 1) self.assertEqual(observed[0][0], "memory_store") self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) @@ -1777,6 +1783,56 @@ def test_record_outcome_uses_a_sanitized_task7_event_receipt(self) -> None: self.assertTrue(validated.data.worked) self.assertFalse(validated.data.idempotent_replay) + def test_transient_writer_contention_is_retryable_database_in_use(self) -> None: + # Pool saturation and SQLite lock timeouts clear on retry (F-020/F-027). + from daem0nmcp.api.v7 import pinned + from daem0nmcp.api.v7.errors import ErrorCode + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError + from daem0nmcp.api.v7.tools import MemoryStoreInput, MemoryStoreOutput + + workspace = Workspace(WORKSPACE_ID, Path.cwd()) + scope = InvocationScope("principal", "session-busy", str(workspace.root)) + gate = _gate() + gate.record_briefing(scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_type": "decision", + "content": "Contention is transient.", + "idempotency_key": "decision-busy-1", + } + token = gate.issue_preflight(scope, "memory_store", target) + handlers = pinned.build_pinned_handlers( + _dependencies( + gate=gate, + scope=scope, + workspace=workspace, + briefing_service=_UnusedService(), + memory_event_writer=_FailingMemoryEventWriter( + RuntimeServiceError("DATABASE_IN_USE") + ), + ) + ) + + request = MemoryStoreInput(**target, preflight_token=token).model_dump() + response = asyncio.run(handlers["memory_store"](**request)) + + validated = MemoryStoreOutput.model_validate(response) + self.assertFalse(validated.ok) + self.assertEqual(ErrorCode.DATABASE_IN_USE, validated.error.code) + self.assertTrue(validated.error.retryable) + self.assertEqual(250, validated.error.retry_after_ms) + # authorize() spent the token before the write, so the documented + # retry is a fresh preflight with the same idempotency_key. + self.assertEqual("memory_preflight", validated.error.remedy.tool) + remedy_target = validated.error.remedy.arguments["target_arguments"] + self.assertEqual("decision-busy-1", remedy_target["idempotency_key"]) + self.assertNotIn("preflight_token", remedy_target) + replayed = MemoryStoreOutput.model_validate( + asyncio.run(handlers["memory_store"](**request)) + ) + self.assertEqual(ErrorCode.TOKEN_REPLAYED, replayed.error.code) + self.assertEqual("memory_preflight", replayed.error.remedy.tool) + def test_record_outcome_maps_runtime_not_found(self) -> None: from daem0nmcp.api.v7 import pinned from daem0nmcp.api.v7.errors import ErrorCode diff --git a/tests/api_v7/test_process_concurrent_writers.py b/tests/api_v7/test_process_concurrent_writers.py new file mode 100644 index 0000000..dc7b88b --- /dev/null +++ b/tests/api_v7/test_process_concurrent_writers.py @@ -0,0 +1,117 @@ +"""Two stdio server processes writing one workspace store each key exactly once.""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from collections.abc import Callable +from contextlib import closing +from pathlib import Path +from typing import Any + +from mcp import ClientSession + +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import call, initialize_workspaces, process_client + +SAME_KEY_STORES = 5 +DISTINCT_KEY_STORES = 10 +LOG_NAMES = ("stdio-server-a.log", "stdio-server-b.log") + + +def _server_log_tails(root: Path, lines: int = 40) -> str: + """Server stderr (tracebacks by correlation ID) for assertion messages.""" + tails = [] + for name in LOG_NAMES: + try: + text = (root / name).read_text(encoding="utf-8", errors="replace") + except OSError as error: + text = f"" + tails.append(f"--- {name} ---\n" + "\n".join(text.splitlines()[-lines:])) + return "\n".join(tails) + + +async def _store_until_ok( + session: ClientSession, + workspace_id: str, + key: str, + content: str, + diagnostics: Callable[[], str], +) -> list[dict[str, Any]]: + """Preflight and store, retrying only what the server marks retryable.""" + scope = {"workspace_id": workspace_id} + arguments = {"record_type": "decision", "content": content, "idempotency_key": key} + observed: list[dict[str, Any]] = [] + deadline = asyncio.get_running_loop().time() + 120 + while True: + response = await call( + session, + "memory_preflight", + {**scope, "target_tool": "memory_store", "target_arguments": arguments}, + ) + if response["ok"]: + token = response["data"]["preflight_token"] + response = await call( + session, + "memory_store", + {**scope, **arguments, "preflight_token": token}, + ) + observed.append(response) + if response["ok"]: + return observed + retryable = ( + response["error"]["code"] == "DATABASE_IN_USE" + and response["error"]["retryable"] is True + ) + assert retryable, (observed, diagnostics()) + assert asyncio.get_running_loop().time() < deadline, (observed, diagnostics()) + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + +async def test_two_stdio_servers_store_each_key_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + scope = {"workspace_id": workspace.workspace_id} + async with ( + process_client(tmp_path, "stdio", log_name=LOG_NAMES[0]) as first, + process_client(tmp_path, "stdio", log_name=LOG_NAMES[1]) as second, + ): + sessions = (first, second) + for session in sessions: + assert (await call(session, "session_brief", scope))["ok"] + stores = [ + ("shared-store-key-0001", "One decision stored from two processes") + ] * SAME_KEY_STORES + [ + (f"distinct-store-key-{index:04d}", f"Distinct decision {index}") + for index in range(DISTINCT_KEY_STORES) + ] + results = await asyncio.gather( + *( + _store_until_ok( + sessions[index % 2], + workspace.workspace_id, + key, + content, + lambda: _server_log_tails(tmp_path), + ) + for index, (key, content) in enumerate(stores) + ) + ) + + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) + record_ids = {observed[-1]["data"]["record"]["record_id"] for observed in results} + assert len(record_ids) == 1 + DISTINCT_KEY_STORES + replays = [observed[-1]["data"]["idempotent_replay"] for observed in results] + assert replays[:SAME_KEY_STORES].count(False) == 1, replays + + database = resolve_active_database(tmp_path / ".daem0nmcp" / "storage").path + with closing(sqlite3.connect(database)) as connection: + per_stream = dict( + connection.execute( + "SELECT stream_id, count(*) FROM memory_events " + "WHERE event_type='memory.created' GROUP BY stream_id" + ).fetchall() + ) + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + assert per_stream == dict.fromkeys(record_ids, 1) + assert records[0] == 1 + DISTINCT_KEY_STORES diff --git a/tests/api_v7/test_process_discovery_surface.py b/tests/api_v7/test_process_discovery_surface.py index 2b3f101..60e9377 100644 --- a/tests/api_v7/test_process_discovery_surface.py +++ b/tests/api_v7/test_process_discovery_surface.py @@ -2,13 +2,14 @@ from __future__ import annotations +import asyncio import importlib.util import pytest from daem0nmcp.database import DatabaseManager from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed @pytest.fixture @@ -132,15 +133,24 @@ async def test_production_graph_discovery_surface(initialized_workspace, transpo {**scope, "entity_id": entity["entity_id"]}, ) assert traced["entity"]["entity_id"] == entity["entity_id"] - recalled = await succeed( - session, - "memory_recall_entity", - {**scope, "entity_name": entity["name"]}, - ) - assert {item["record_id"] for item in recalled["items"]} & { - first["record_id"], - second["record_id"], - } + # The lexical projection may still be catching up with the link and + # backfill events; recall reports that as retryable until it has. + recall_deadline = asyncio.get_running_loop().time() + 30 + while True: + recalled = await call( + session, + "memory_recall_entity", + {**scope, "entity_name": entity["name"]}, + ) + if recalled["ok"] and { + item["record_id"] for item in recalled["data"]["items"] + } & {first["record_id"], second["record_id"]}: + break + if not recalled["ok"]: + assert recalled["error"]["code"] == "DATABASE_IN_USE", recalled + assert recalled["error"]["retryable"], recalled + assert asyncio.get_running_loop().time() < recall_deadline, recalled + await asyncio.sleep(0.25) graph = await succeed( session, "knowledge_graph_get", diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index c6f1ac2..7bfc48b 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -409,6 +409,8 @@ async def snapshot(_workspace, **limits): ) self.assertEqual(len(calls), 1) + # Guidance never shows git changes, so it must not spawn git for them. + self.assertIs(calls[0]["include_git_changes"], False) self.assertEqual(result["records"], [relevant_failure]) self.assertEqual(result["rules"], [relevant_rule]) self.assertIn( @@ -600,5 +602,54 @@ def build_server(_surface, **options): self.assertEqual([False], observed) +async def test_memory_preflight_spawns_no_subprocess(tmp_path): + """Preflight guidance must not pay for git (F-040); session_brief still does.""" + import subprocess + + from fastmcp import Client + + from daem0nmcp.api.v7.production import create_v7_server + from tests.api_v7.process_client import initialize_workspaces + + [workspace] = await initialize_workspaces((tmp_path,)) + spawned: list[list[str]] = [] + + def record_spawn(argv, *args, **kwargs): + spawned.append(list(argv)) + raise FileNotFoundError(argv[0]) + + server = create_v7_server( + "stdio", + settings=Settings( + project_root=str(tmp_path), + workspace_roots=[str(tmp_path)], + dream_enabled=False, + ), + environ={}, + ) + scope = {"workspace_id": workspace.workspace_id} + async with Client(server) as client: + with patch.object(subprocess, "Popen", side_effect=record_spawn): + brief = await client.call_tool("session_brief", scope, raise_on_error=False) + assert brief.structured_content["ok"], brief.structured_content + assert any(argv[0] == "git" for argv in spawned), spawned + spawned.clear() + preflight = await client.call_tool( + "memory_preflight", + { + **scope, + "target_tool": "memory_store", + "target_arguments": { + "record_type": "decision", + "content": "Preflight must not spawn processes.", + "idempotency_key": "preflight-no-subprocess-0001", + }, + }, + raise_on_error=False, + ) + assert preflight.structured_content["ok"], preflight.structured_content + assert spawned == [] + + if __name__ == "__main__": unittest.main() diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index b97fab7..5805f68 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -715,8 +715,10 @@ def resolve(_workspace: Workspace) -> ResolvedActiveDatabase: self.assertFalse(read.done()) self.assertEqual(pool.in_flight, 1) release.set() - with self.assertRaises(ResourceRepositoryError): + with self.assertRaises(ResourceRepositoryError) as caught: await read + # An overrun read that did not itself fail is retryable contention. + self.assertEqual(caught.exception.code, "DATABASE_IN_USE") self.assertEqual(pool.in_flight, 0) finally: release.set() @@ -835,6 +837,32 @@ def read_records(*args, **kwargs): ) self.assertEqual(snapshot.workspace_statistics["active_context"], 2) + async def test_snapshot_skips_git_subprocesses_when_changes_are_unused( + self, + ) -> None: + # Preflight guidance paid two git subprocesses per call for data it + # discarded; under load that pushed reads past their deadline. + from daem0nmcp.api.v7.resource_repository import SQLiteResourceRepository + + repository = SQLiteResourceRepository( + lambda _workspace: self.fixture.resolved, + clock=lambda: NOW, + ) + git_reads: list[object] = [] + + def read_git_changes(workspace): + git_reads.append(workspace) + return [] + + repository._read_git_changes_sync = read_git_changes + limits = {"warning_limit": 1, "failure_limit": 1, "active_context_limit": 1} + skipped = await repository.read_briefing_snapshot( + self.fixture.workspace, include_git_changes=False, **limits + ) + self.assertEqual((skipped.git_changes, git_reads), ([], [])) + await repository.read_briefing_snapshot(self.fixture.workspace, **limits) + self.assertEqual(len(git_reads), 1) + async def test_briefing_sections_share_one_sqlite_read_snapshot(self) -> None: # A canonical writer may commit while holding the same shared generation # lock. Every section of one brief must still observe one SQLite snapshot. diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index ffffc8d..2f63c4f 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -332,6 +332,7 @@ async def test_outcome_updates_existing_workspace_record_and_replays( async def test_cancelled_worker_retains_capacity_and_leaves_no_write(self) -> None: """Cancelling an asyncio waiter must not over-admit or commit later.""" + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError from daem0nmcp.bounded_workers import BoundedWorkerBusyError clock_entered = threading.Event() @@ -347,11 +348,13 @@ def blocked_clock() -> datetime: await asyncio.to_thread(clock_entered.wait, 1.0) first.cancel() await asyncio.sleep(0) - with self.assertRaises(BoundedWorkerBusyError): + with self.assertRaises(RuntimeServiceError) as busy: await writer.store( self.workspace, self._store_command(idempotency_key="runtime-store-overflow"), ) + self.assertEqual("DATABASE_IN_USE", busy.exception.code) + self.assertIsInstance(busy.exception.__cause__, BoundedWorkerBusyError) release_clock.set() with self.assertRaises(asyncio.CancelledError): await first diff --git a/tests/test_cli_retrieval.py b/tests/test_cli_retrieval.py index 057f712..5c60a42 100644 --- a/tests/test_cli_retrieval.py +++ b/tests/test_cli_retrieval.py @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from tests.test_migrate_v7 import _create_legacy_database @@ -157,6 +158,66 @@ def test_registered_secondary_workspace_opens_its_own_active_database(self): self.assertEqual(1, len(lexical)) self.assertEqual(second_id, payload["workspace_id"]) + def test_compact_projections_drains_superseded_generations_offline(self): + """Databases bloated before the GC existed need a one-shot drain.""" + + import sqlite3 + + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.storage_activation import resolve_active_database + + with tempfile.TemporaryDirectory() as raw: + root, workspace_id = self._migrated_fixture(raw) + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(9): + LexicalProjectionBuilder(connection).rebuild(workspace_id) + superseded = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical' AND status<>'active'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(9, superseded) + + result = self._run( + root, + "compact-projections", + "--workspace-id", + workspace_id, + ) + + self.assertEqual(0, result.returncode, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(8, payload["collected"]["lexical"]) + self.assertLess( + payload["superseded_manifests_after"], + payload["superseded_manifests_before"], + ) + self.assertTrue(payload["vacuumed"]) + self.assertNotIn(str(root), result.stdout) + connection = sqlite3.connect(database) + try: + remaining = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical'" + ).fetchone()[0] + tables = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND name GLOB 'retrieval_fts_*_g*' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(2, remaining) + self.assertEqual(2, tables) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_event_store.py b/tests/test_event_store.py index 5d2144d..6e3066a 100644 --- a/tests/test_event_store.py +++ b/tests/test_event_store.py @@ -257,6 +257,103 @@ def test_expected_version_is_exactly_idempotent_or_conflicts(self): self.connection.execute("SELECT count(*) FROM memory_events").fetchone()[0], ) + def test_only_integrity_errors_on_insert_are_stream_conflicts(self): + """A lock or I/O failure must not be reported as a retryable conflict.""" + command_type, store_type = self._api() + from daem0nmcp.event_store import EventStreamConflict + + connection = self.connection + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO memory_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = command_type( + workspace_id=self.workspace_id, + stream_id="mem_" + "c" * 64, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + payload={"record": self._state()}, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + store_type(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + store_type(failing).append_and_project(command) + connection.rollback() + + def test_only_integrity_errors_on_governance_insert_are_conflicts(self): + """The governance append follows the same conflict rule as memory.""" + from daem0nmcp.event_store import ( + EventStreamConflict, + GovernanceEventCommand, + GovernanceEventStore, + ) + from daem0nmcp.migrations.schema import MIGRATIONS + from daem0nmcp.schema_version import CURRENT_SCHEMA_VERSION + + connection = sqlite3.connect(":memory:") + self.addCleanup(connection.close) + connection.execute("PRAGMA foreign_keys=ON") + for version in range(16, CURRENT_SCHEMA_VERSION + 1): + for statement in next(m for m in MIGRATIONS if m[0] == version)[2]: + connection.execute(statement) + connection.commit() + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO governance_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = GovernanceEventCommand( + workspace_id=self.workspace_id, + stream_id="rule_" + "d" * 64, + stream_kind="rule", + event_type="rule.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + expected_stream_version=1, + payload={ + "rule_id": "rule_" + "d" * 64, + "trigger": "conflict mapping", + "must_do": [], + "must_not": [], + "ask_first": [], + "warnings": [], + "priority": 1, + "enabled": True, + "created_at_us": 1, + "updated_at_us": 1, + }, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + GovernanceEventStore(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + GovernanceEventStore(failing).append_and_project(command) + connection.rollback() + def test_projection_failure_rolls_back_event_with_savepoint(self): """No immutable event may survive a failed projection in caller transaction.""" command_type, store_type = self._api() diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index cb4969d..5710696 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -1320,8 +1320,9 @@ def test_reactivation_preserves_only_the_previous_active_lexical_generation(self self.assertEqual("reactivate", reactivated.action) active = sqlite3.connect(resolve_active_database(storage).path) try: + # Generation 1 is superseded twice over and is collected. self.assertEqual( - [(1, "ready"), (2, "ready"), (3, "active")], + [(2, "ready"), (3, "active")], active.execute( "SELECT generation,status FROM projection_manifests " "WHERE projection_name='lexical' ORDER BY generation" diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py new file mode 100644 index 0000000..64014fb --- /dev/null +++ b/tests/test_retrieval_projection_growth.py @@ -0,0 +1,570 @@ +"""Superseded projection generations are garbage-collected (KD-5, F-001). + +Every write rebuilds the local projections into a new generation. Without GC +each rebuild left a full-corpus FTS5 table, its document rows and a manifest +behind, so storage grew quadratically with the number of memories. +""" + +from __future__ import annotations + +import re +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +WORKSPACE_ID = "ws_0123456789abcdef01234567" +_NO_OPTIONAL_PROFILES = { + "local": "disabled", + "models-local": "disabled", + "graph": "disabled", +} +_LOCAL = ("lexical", "procedure", "outcome", "temporal") +# Spelled out here on purpose: importing the production mapping would make a +# shrunken GENERATION_TABLES pass this file. +_GENERATION_TABLES = { + "lexical": "retrieval_documents", + "outcome": "record_outcome_view", + "procedure": "record_procedures", +} + + +def _record(content: str, *, procedure: bool = False) -> dict[str, object]: + return { + "record_type": "procedure" if procedure else "decision", + "legacy_type": None, + "content": content, + "rationale": None, + "context": {"steps": ["Check state.", "Apply change."]} if procedure else {}, + "tags": ["growth"], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "growth-test", + "source_model": None, + "deleted_at_us": None, + } + + +class ProjectionGenerationGrowthTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + from daem0nmcp.migrations.schema import MIGRATIONS + + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "growth.sqlite3" + self.connection = sqlite3.connect(self.path) + self.connection.execute("PRAGMA foreign_keys=ON") + # Pre-PR-8 databases on POSIX still use the rollback journal, where + # readers block a committing writer; GC must stay safe there. + self.connection.execute("PRAGMA journal_mode=DELETE") + # Durability is irrelevant here; skipping fsync keeps the hundreds of + # seeding rebuilds fast on slow CI disks. + self.connection.execute("PRAGMA synchronous=OFF") + for version, _description, statements in MIGRATIONS: + if 16 <= version <= 18: + for statement in statements: + self.connection.execute(statement) + self.connection.commit() + self._sequence = 0 + + def tearDown(self) -> None: + self.connection.close() + self._directory.cleanup() + + def _record_outcome(self, record_id: str) -> None: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.outcome_recorded", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": { + **_record("growth memory number 1", procedure=True), + "outcome": "the generation was collected", + "worked": True, + } + }, + ) + ) + self.connection.commit() + + def _append(self, *, procedure: bool = False) -> str: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + record_id = f"mem_{self._sequence:064x}" + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": _record( + f"growth memory number {self._sequence}", procedure=procedure + ) + }, + ) + ) + self.connection.commit() + return record_id + + def _rebuild(self, projection: str) -> None: + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.retrieval.specialized_projection import ( + SpecializedProjectionBuilder, + ) + + if projection == "lexical": + LexicalProjectionBuilder(self.connection).rebuild(WORKSPACE_ID) + else: + SpecializedProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID, projection, force=True + ) + + def _manifests(self, projection: str) -> dict[int, str]: + return { + int(row[0]): str(row[1]) + for row in self.connection.execute( + "SELECT generation,status FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=?", + (WORKSPACE_ID, projection), + ) + } + + def _fts_generations(self, projection: str) -> set[int]: + prefix = ( + "retrieval_procedure_fts_" + if projection == "procedure" + else ("retrieval_fts_") + ) + pattern = re.compile(re.escape(prefix + WORKSPACE_ID[3:]) + r"_g(\d+)") + return { + int(match[1]) + for (name,) in self.connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + if (match := pattern.fullmatch(str(name))) is not None + } + + def _active(self, projection: str) -> int: + [generation] = [ + generation + for generation, status in self._manifests(projection).items() + if status == "active" + ] + return generation + + async def test_spaced_writes_keep_generations_bounded(self) -> None: + from daem0nmcp.retrieval.runtime import drain_projection_jobs + + first = self._append(procedure=True) + self._record_outcome(first) + for projection in _LOCAL: + self._rebuild(projection) + writes = 20 + for index in range(writes): + self._append(procedure=index % 5 == 0) + # Every job is due immediately: this is the interactive case of + # writes spaced further apart than the coalescing grace. + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runs = await drain_projection_jobs( + self.path, + config=SimpleNamespace(), + max_jobs=8, + # The real drain: every local projection rebuilds, so the + # specialized GC call site runs on the production path. + include_optional=True, + capability_statuses=_NO_OPTIONAL_PROFILES, + ) + self.assertEqual( + {"succeeded"}, + {run.status for run in runs}, + [run.reason for run in runs], + ) + self.assertEqual( + set(_LOCAL), + {name for run in runs for name in run.projections}, + ) + + for projection in _LOCAL: + manifests = self._manifests(projection) + self.assertLessEqual(len(manifests), 2, (projection, manifests)) + self.assertEqual(1, list(manifests.values()).count("active"), projection) + for projection in ("lexical", "procedure"): + tables = self._fts_generations(projection) + self.assertLessEqual(len(tables), 2, (projection, tables)) + self.assertEqual(tables, set(self._manifests(projection)), projection) + live = self.connection.execute( + "SELECT COUNT(*) FROM memory_records WHERE workspace_id=?", + (WORKSPACE_ID,), + ).fetchone()[0] + self.assertEqual(writes + 1, live) + documents = self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents" + ).fetchone()[0] + self.assertLessEqual(documents, 2 * live) + self._assert_generation_rows_collected() + # The stale marker every write sets clears once the rebuild runs, and + # GC never leaves it stuck on an active manifest. + self.assertEqual( + [], + self.connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active' AND (" + "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " + "OR json_extract(details_json,'$.rebuild_required_at_us') IS NOT NULL)", + (WORKSPACE_ID,), + ).fetchall(), + ) + + def _assert_generation_rows_collected(self) -> None: + """Every generation table keeps rows only for surviving manifests.""" + + for projection, table in _GENERATION_TABLES.items(): + rows = { + int(row[0]) + for row in self.connection.execute( + f'SELECT DISTINCT projection_generation FROM "{table}" ' + "WHERE workspace_id=?", + (WORKSPACE_ID,), + ) + } + self.assertTrue(rows, f"{table} has no rows to prove collection") + self.assertLessEqual(rows, set(self._manifests(projection)), table) + + def test_stale_backlog_shrinks_five_generations_per_activation(self) -> None: + self._append(procedure=True) + for projection in ("lexical", "procedure"): + with self.subTest(projection=projection): + # Seed 120 superseded generations the way a pre-GC database + # accumulated them. + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(121): + self._rebuild(projection) + self.assertEqual(121, len(self._manifests(projection))) + self.assertEqual(121, len(self._fts_generations(projection))) + + previous = 120 + while previous > 1: + self._rebuild(projection) + stale = sum( + status != "active" + for status in self._manifests(projection).values() + ) + self.assertEqual(max(1, previous + 1 - 5), stale) + previous = stale + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests)) + self.assertEqual(set(manifests), self._fts_generations(projection)) + active = self._active(projection) + self.assertEqual({active - 1, active}, set(manifests)) + + def test_failed_or_unactivated_generation_never_evicts_last_active(self) -> None: + from daem0nmcp.event_store import deterministic_id + + self._append() + self._rebuild("lexical") + self._rebuild("lexical") + previous_active = self._active("lexical") + template = self.connection.execute( + "SELECT source_event_count,source_event_root_hash,row_count," + "builder_version FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, previous_active), + ).fetchone() + # Generations just below the next one that never became active: an + # arithmetic "keep the one below the new generation" rule would keep + # these and evict the generation readers were actually using. + for offset, status in ((1, "failed"), (2, "ready")): + generation = previous_active + offset + self.connection.execute( + "INSERT INTO projection_manifests (manifest_id,workspace_id," + "projection_name,generation,projection_version,status," + "source_event_count,source_event_root_hash,row_count," + "builder_version,details_json,started_at_us) " + "VALUES (?,?,'lexical',?,1,?,?,?,?,?,'{}',1)", + ( + deterministic_id("prj", "growth-test", generation), + WORKSPACE_ID, + generation, + status, + *template, + ), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + active = self._active("lexical") + self.assertEqual(previous_active + 3, active) + self.assertEqual({previous_active: "ready", active: "active"}, manifests) + self.assertEqual({previous_active, active}, self._fts_generations("lexical")) + + def test_gc_blocked_by_an_open_reader_does_not_fail_the_rebuild(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + reader = sqlite3.connect(self.path, check_same_thread=False) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + # A reader that looked up an old generation is mid-query when + # the activation's GC starts. + reader.execute("BEGIN") + reader.execute( + f'SELECT COUNT(*) FROM "{lexical_fts_table_name(WORKSPACE_ID, oldest)}"' + ).fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + self.assertEqual([0], collected) + self.assertIn(oldest, self._manifests("lexical")) + self.assertIn(oldest, self._fts_generations("lexical")) + self.assertFalse(self.connection.in_transaction) + finally: + reader.close() + + # The next activation retries and catches up. + self._rebuild("lexical") + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertLessEqual(len(self._manifests("lexical")), 2) + + def test_gc_in_wal_mode_collects_under_an_open_reader(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self.connection.execute("PRAGMA journal_mode=WAL") + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + table = lexical_fts_table_name(WORKSPACE_ID, oldest) + reader = sqlite3.connect(self.path) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + reader.execute("BEGIN") + reader.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + # WAL readers do not block the drop, and the reader's snapshot + # still answers from the generation being collected. + self.assertEqual([1], collected) + self.assertEqual( + 1, len(reader.execute(f'SELECT * FROM "{table}" LIMIT 1').fetchall()) + ) + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertNotIn(oldest, self._fts_generations("lexical")) + finally: + reader.close() + + def test_repair_left_rebuild_required_generations_are_collected(self) -> None: + self._append() + self._rebuild("lexical") + repaired = self._active("lexical") + # verify-v7 --repair-projections demotes the active generation and + # rebuilds in one transaction, so its GC is a no-op. + self.connection.execute( + "UPDATE projection_manifests SET status='rebuild_required' " + "WHERE workspace_id=? AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, repaired), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + self.assertNotIn(repaired, manifests) + self.assertEqual(set(manifests), self._fts_generations("lexical")) + self.assertEqual( + 0, + self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + "AND projection_generation=?", + (WORKSPACE_ID, repaired), + ).fetchone()[0], + ) + + def test_an_undroppable_generation_does_not_wedge_the_rest(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + with patch.object( + projections, "collect_superseded_generations", return_value=0 + ): + for _ in range(4): + self._rebuild("lexical") + generations = sorted(self._manifests("lexical")) + damaged = generations[0] + original = projections._drop_generation + + def refuse_oldest( + connection, workspace_id, projection, manifest_id, generation + ): + if generation == damaged: + raise sqlite3.OperationalError("malformed database schema") + return original( + connection, workspace_id, projection, manifest_id, generation + ) + + with patch.object(projections, "_drop_generation", refuse_oldest): + collected = projections.collect_superseded_generations( + self.connection, WORKSPACE_ID, "lexical" + ) + + manifests = self._manifests("lexical") + # The damaged generation is skipped, not retried forever, and every + # later victim is still collected. + self.assertGreaterEqual(collected, 1) + self.assertIn(damaged, manifests) + self.assertIn(damaged, self._fts_generations("lexical")) + self.assertNotIn(generations[1], manifests) + self.assertNotIn(generations[1], self._fts_generations("lexical")) + self.assertIsNotNone(lexical_fts_table_name(WORKSPACE_ID, damaged)) + + def test_offline_compaction_drains_a_backlog_and_reclaims_the_file(self) -> None: + from daem0nmcp.retrieval.operations import compact_projections + + self._append(procedure=True) + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(30): + for projection in ("lexical", "procedure"): + self._rebuild(projection) + self.assertEqual(30, len(self._manifests("lexical"))) + before = self.path.stat().st_size + + payload = compact_projections(self.connection, WORKSPACE_ID) + + self.assertEqual(28, payload["collected"]["lexical"]) + self.assertEqual(28, payload["collected"]["procedure"]) + self.assertTrue(payload["vacuumed"]) + self.assertEqual(58, payload["superseded_manifests_before"]) + self.assertEqual(2, payload["superseded_manifests_after"]) + for projection in ("lexical", "procedure"): + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests), projection) + self.assertEqual(set(manifests), self._fts_generations(projection)) + self.assertLess(self.path.stat().st_size, before) + + def test_lock_contention_is_retried_not_dead_lettered(self) -> None: + from daem0nmcp.retrieval.jobs import ProjectionJobRunner + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._append() + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runner_connection = sqlite3.connect(self.path, timeout=0.1) + blocker = sqlite3.connect(self.path, check_same_thread=False) + lexical = LexicalProjectionBuilder(runner_connection) + + def contended_rebuild(workspace_id: str): + blocker.execute("BEGIN IMMEDIATE") + try: + return lexical.rebuild(workspace_id) + finally: + blocker.rollback() + + try: + run = ProjectionJobRunner( + runner_connection, builders={"lexical": contended_rebuild} + ).run_once() + assert run is not None + self.assertEqual("DATABASE_IN_USE", run.reason) + self.assertEqual("queued", run.status) + finally: + blocker.close() + runner_connection.close() + + +class GenerationGcUnitTests(unittest.TestCase): + def test_gc_inside_a_caller_transaction_is_a_no_op(self) -> None: + from daem0nmcp.retrieval.projections import collect_superseded_generations + + connection = sqlite3.connect(":memory:") + try: + connection.execute("CREATE TABLE t(x)") + connection.execute("INSERT INTO t VALUES (1)") + self.assertTrue(connection.in_transaction) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "lexical") + ) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "graph") + ) + finally: + connection.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_v7.py b/tests/test_verify_v7.py index c50ed20..3ad9705 100644 --- a/tests/test_verify_v7.py +++ b/tests/test_verify_v7.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch import pytest @@ -198,6 +199,32 @@ def test_verify_v7_replays_every_authority_domain(tmp_path: Path) -> None: assert report["checks"]["active_pointer"]["generation"] == 1 +def test_verify_v7_reports_superseded_generation_counts(tmp_path: Path) -> None: + """A GC that has stopped making progress must be visible without logs.""" + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + storage, database = _storage(tmp_path) + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(3): + LexicalProjectionBuilder(connection).rebuild(WORKSPACE_ID) + connection.commit() + + report = verify_v7(storage, WORKSPACE_ID) + + inventory = report["checks"]["projection_generations"] + assert inventory["ok"] is True + # Three generations: one active, two superseded and uncollected. + assert inventory["lexical"] == 2 + assert inventory["outcome"] == 0 + # Only the partitions themselves, never their FTS5 shadow tables. + assert inventory["fts_partitions"] == 3 + + def test_verify_v7_accepts_intact_active_code_generation(tmp_path: Path) -> None: storage, database = _storage(tmp_path) _build_code_projection(database) From f376ff384828580d7018fc2bcffe644666a99de7 Mon Sep 17 00:00:00 2001 From: Devil Date: Sun, 20 Sep 2026 15:08:48 -0400 Subject: [PATCH 70/72] test: quiesce the brief's drains before the preflight window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI run 35520333257 (ubuntu 3.11) saw `memory_preflight` "spawn" the Leiden graph worker. It does not: the recorder patches `subprocess.Popen` process-wide, and the spawn came from `daem0nmcp-projection-runtime_0` — the projection drain the *brief's* write scheduled, which with the graph profile now auto-enabled rebuilds communities and runs Leiden in its own thread. Traced it: `_drain_projection_jobs_sync` -> `jobs.run_once` -> `GraphProjectionBuilder.rebuild` -> `_community_seeds` -> `run_leiden_bounded`. Nothing on the preflight path reaches it, so PR 4's guarantee holds and preflight latency is unaffected. The fix closes the overlap rather than narrowing the assertion: the test awaits the drains the brief scheduled before clearing the recorder, so the window is quiet by construction and `assert spawned == []` stays literal — no subprocess of any kind, not "no git specifically", and with the graph profile left enabled. Deterministic across 14 runs where the unquiesced window leaked on 1 of 2. Verified it still fails on a regression: injecting a `Popen` into the per-request guidance closure fails the test both for `git` (F-040's original regression) and for any other worker. Co-Authored-By: Claude Opus 5 --- tests/api_v7/test_production.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index d878d19..8543d66 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -649,9 +649,12 @@ async def test_memory_preflight_spawns_no_subprocess(tmp_path): from fastmcp import Client from daem0nmcp.api.v7.production import create_v7_server + from daem0nmcp.retrieval.runtime import await_projection_job_drains + from daem0nmcp.storage_activation import resolve_active_database from tests.api_v7.process_client import initialize_workspaces [workspace] = await initialize_workspaces((tmp_path,)) + database = resolve_active_database(tmp_path / ".daem0nmcp" / "storage").path spawned: list[list[str]] = [] def record_spawn(argv, *args, **kwargs): @@ -673,6 +676,15 @@ def record_spawn(argv, *args, **kwargs): brief = await client.call_tool("session_brief", scope, raise_on_error=False) assert brief.structured_content["ok"], brief.structured_content assert any(argv[0] == "git" for argv in spawned), spawned + # ``Popen`` is patched process-wide, so anything running + # concurrently lands in ``spawned`` too. The brief's write + # schedules a projection drain, and with the graph profile + # installed that drain rebuilds communities on its own thread + # and spawns the Leiden worker. Let the drains it scheduled + # finish before the window opens, so the assertion below can + # stay literal -- no subprocess at all -- instead of being + # narrowed to git or excused for a thread name. + await await_projection_job_drains((database,)) spawned.clear() preflight = await client.call_tool( "memory_preflight", From cf5bba52c396f7a7f7081291dc99a9a22b457d81 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 20:40:39 +0000 Subject: [PATCH 71/72] fix: let a finalize retried after DATABASE_IN_USE release its dead lease A finalize claims a durable lease, so a concurrent writer that holds the SQLite write lock past the busy timeout fails the attempt as the retryable DATABASE_IN_USE and then fails its cleanup release the same way. The release error was suppressed, so the lease stayed for its full 15 minutes and every retry the error invites hit the live-lease branch of claim_import_finalization: a non-retryable TASK_REQUIRED. Keep a lease whose release failed, and release it before the next finalize of the session claims. A still-held lock then fails that release as DATABASE_IN_USE, so the client keeps retrying until the writer finishes. A live concurrent finalizer is unaffected and still gets TASK_REQUIRED. Seen in CI on #69: test_process_portability's retried finalize got TASK_REQUIRED on ubuntu-latest / Python 3.12. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UBLLzyphecQABpqVorHunX --- daem0nmcp/api/v7/operations.py | 49 +++++++++++- tests/api_v7/test_portable_projections.py | 91 +++++++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index 38964d2..456b377 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -157,6 +157,16 @@ thread_name_prefix="daem0nmcp-v7-core", ) +# Finalization leases whose attempt has ended without deleting them, keyed by +# storage and import session. A concurrent writer that outlasts the busy +# timeout fails the attempt as DATABASE_IN_USE and then its cleanup release +# too; the lease would otherwise turn every retry that DATABASE_IN_USE invites +# into TASK_REQUIRED until it expires. The ended attempt no longer renews or +# uses its lease, so the next finalize of the session completes the release +# before it claims. +_ABANDONED_FINALIZATION_LEASES: dict[tuple[str, str], ImportFinalizationLease] = {} +_ABANDONED_FINALIZATION_LEASES_LOCK = threading.Lock() + class CoreOperationError(RuntimeError): """Sanitized operation failure understood by the shared v7 router.""" @@ -1200,6 +1210,35 @@ def _event_command(event: Mapping[str, Any]) -> EventCommand: ) +def _abandon_finalization_lease(storage: Path, lease: ImportFinalizationLease) -> None: + with _ABANDONED_FINALIZATION_LEASES_LOCK: + _ABANDONED_FINALIZATION_LEASES[(str(storage), lease.session_id)] = lease + + +def _release_abandoned_finalization( + connection: sqlite3.Connection, + storage: Path, + session_id: str, + *, + now: datetime, +) -> None: + """Finish an ended attempt's lease release before claiming the session. + + A still-held lock raises here and reaches the caller as DATABASE_IN_USE, + so the lease stays recorded for the next retry. + """ + + key = (str(storage), session_id) + with _ABANDONED_FINALIZATION_LEASES_LOCK: + abandoned = _ABANDONED_FINALIZATION_LEASES.get(key) + if abandoned is None: + return + release_import_finalization(connection, storage, abandoned, now=now) + with _ABANDONED_FINALIZATION_LEASES_LOCK: + if _ABANDONED_FINALIZATION_LEASES.get(key) is abandoned: + del _ABANDONED_FINALIZATION_LEASES[key] + + def _import_v2_sync( dependencies: CoreOperationDependencies, workspace: Workspace, @@ -1294,6 +1333,12 @@ def _import_v2_sync( "skipped": receipt.imported + receipt.skipped, } ) + _release_abandoned_finalization( + connection, + storage, + session_id, + now=dependencies.clock(), + ) lease = claim_import_finalization( connection, storage, @@ -1554,13 +1599,15 @@ def finalization_checkpoint() -> None: ) from exc finally: if lease is not None and not committed: - with suppress(Exception): + try: release_import_finalization( connection, storage, lease, now=dependencies.clock(), ) + except Exception: + _abandon_finalization_lease(storage, lease) def _import_sync( diff --git a/tests/api_v7/test_portable_projections.py b/tests/api_v7/test_portable_projections.py index 417d12d..7ab99fe 100644 --- a/tests/api_v7/test_portable_projections.py +++ b/tests/api_v7/test_portable_projections.py @@ -2016,3 +2016,94 @@ def test_a_concurrent_writer_is_reported_as_retryable_not_as_a_bad_bundle(): _portable_failure_code(ValueError("bad page"), "workspace_import") == "IMPORT_INVALID" ) + + +async def test_retry_after_a_contended_finalize_releases_its_abandoned_lease( + tmp_path: Path, +): + """A finalize retried after DATABASE_IN_USE must not meet its own dead lease. + + The failed attempt claims a durable lease, then a concurrent writer holds the + SQLite write lock past the busy timeout, so both the attempt and its cleanup + release report "database is locked". The retry that DATABASE_IN_USE invites + must finish the import, not answer the non-retryable TASK_REQUIRED until the + abandoned lease expires. + """ + from daem0nmcp.api.v7 import operations as operations_module + + source = _fixture(tmp_path / "source") + target = _fixture(tmp_path / "target") + source.append("contended finalize", occurred_at_us=100, recorded_at_us=200) + pages = await _all_pages( + build_core_operations(_dependencies(source))["workspace_export"], + source, + include_legacy_projection=False, + ) + importer = build_core_operations(_dependencies(target))["workspace_import"] + session_id = None + for index, page in enumerate(pages): + staged = await importer( + workspace=target.workspace, + request=_request( + "workspace_import", + workspace_id=WORKSPACE_ID, + bundle=page.model_dump(mode="python"), + import_session_id=session_id, + finalize=False, + idempotency_key=f"contended-stage-{index:04d}", + preflight_token="token_value_for_test", + ), + ) + session_id = staged.import_session_id + finalize_request = _request( + "workspace_import", + workspace_id=WORKSPACE_ID, + bundle=None, + import_session_id=session_id, + finalize=True, + idempotency_key="contended-finalize-0001", + preflight_token="token_value_for_test", + ) + original_prepare = operations_module.prepare_vector_candidate + original_release = operations_module.release_import_finalization + calls = {"prepare": 0, "release": 0} + + def contended_prepare(*args, **kwargs): + calls["prepare"] += 1 + if calls["prepare"] == 1: + raise sqlite3.OperationalError("database is locked") + return original_prepare(*args, **kwargs) + + def contended_release(*args, **kwargs): + calls["release"] += 1 + if calls["release"] == 1: + raise sqlite3.OperationalError("database is locked") + return original_release(*args, **kwargs) + + with ( + patch.object( + operations_module, + "prepare_vector_candidate", + side_effect=contended_prepare, + ), + patch.object( + operations_module, + "release_import_finalization", + side_effect=contended_release, + ), + ): + with pytest.raises(CoreOperationError, match="DATABASE_IN_USE"): + await importer(workspace=target.workspace, request=finalize_request) + result = await importer(workspace=target.workspace, request=finalize_request) + assert result.status == "succeeded" + assert result.imported == 1 + connection = sqlite3.connect(target.database) + try: + leases = connection.execute( + "SELECT COUNT(*) FROM portable_transfer_finalization_leases " + "WHERE session_id=?", + (session_id,), + ).fetchone()[0] + finally: + connection.close() + assert leases == 0 From 42f929ca1dd223d7f55fabc6966e8bbde5614f35 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 20:40:39 +0000 Subject: [PATCH 72/72] test: retry DATABASE_IN_USE in the path-policy import round trip The graph profile's projection drains can hold the write lock past the busy timeout, and the server then answers the import with the retryable DATABASE_IN_USE. The path-policy round trip asserted success on the first call; retry the preflight and import within a bounded deadline, as the portability test does. Seen in CI on #69 on windows-latest / Python 3.10. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UBLLzyphecQABpqVorHunX --- tests/api_v7/test_path_policy.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py index 1e6ac88..e68af7c 100644 --- a/tests/api_v7/test_path_policy.py +++ b/tests/api_v7/test_path_policy.py @@ -8,6 +8,7 @@ from __future__ import annotations +import asyncio import copy import hashlib import sqlite3 @@ -790,6 +791,25 @@ async def _export_pages(invoke, workspace_id: str) -> list[dict[str, Any]]: return pages +async def _import_call(invoke, workspace_id: str, arguments: dict) -> dict[str, Any]: + """Preflight and run one import call, waiting out a held write lock. + + The graph profile's projection drains can hold the SQLite write lock past + the server's busy timeout, which the preflight and the import report as the + retryable DATABASE_IN_USE. A real client retries those; so does this test, + within a bounded deadline and a bounded number of attempts. + """ + + deadline = time.monotonic() + 30 + for _attempt in range(8): + result = await _target_call(invoke, workspace_id, "workspace_import", arguments) + if result["ok"] or result["error"]["code"] != "DATABASE_IN_USE": + return result + assert time.monotonic() < deadline, result["error"] + await asyncio.sleep(0.5) + raise AssertionError("the import stayed contended for 8 attempts") + + async def _import_pages(invoke, workspace_id: str, pages) -> dict[str, Any]: import_id = None key = "mech-import-0001" @@ -797,11 +817,11 @@ async def _import_pages(invoke, workspace_id: str, pages) -> dict[str, Any]: arguments = {"bundle": page, "finalize": False, "idempotency_key": key} if import_id is not None: arguments["import_session_id"] = import_id - staged = await _target_call(invoke, workspace_id, "workspace_import", arguments) + staged = await _import_call(invoke, workspace_id, arguments) assert staged["ok"], staged["error"] import_id = staged["data"]["import_session_id"] finish = {"import_session_id": import_id, "finalize": True, "idempotency_key": key} - done = await _target_call(invoke, workspace_id, "workspace_import", finish) + done = await _import_call(invoke, workspace_id, finish) assert done["ok"], done["error"] return done["data"]