diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..607e8e11 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Governed prompt templates (ISO 42001 A.5.2). Any change to a prompt asset +# requires sign-off from a designated Prompt Engineer / Product Architect. +# See kai_mcp_solution_server/PROMPT_GOVERNANCE.md. +/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/ @konveyor/kai-prompt-reviewers +/kai_mcp_solution_server/scripts/prompts_version.py @konveyor/kai-prompt-reviewers diff --git a/.github/workflows/prompt-validation.yml b/.github/workflows/prompt-validation.yml new file mode 100644 index 00000000..1d790ae5 --- /dev/null +++ b/.github/workflows/prompt-validation.yml @@ -0,0 +1,48 @@ +name: Prompt Template Validation (ISO 42001 A.5.2) + +# Governance gate for the versioned prompt template set in the solution server. +# Runs whenever a governed prompt asset, the manifest, the validation script, the +# render registry, or this workflow changes. Required-status + CODEOWNERS review +# for the prompts directory are documented in PROMPT_GOVERNANCE.md. +on: + pull_request: + paths: + - "kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/**" + - "kai_mcp_solution_server/scripts/prompts_version.py" + - "kai_mcp_solution_server/tests/prompts/**" + - ".github/workflows/prompt-validation.yml" + workflow_dispatch: + +jobs: + validate-prompts: + name: Validate prompt templates + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./kai_mcp_solution_server + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + version: "latest" + + - name: Install dependencies + run: uv sync + + # Syntactic verification + version governance: every template declared in + # manifest.yaml (and vice-versa), checksums match, Jinja parses, declared + # variables match the template's referenced variables, no leftover f-strings. + - name: Syntactic & manifest validation + run: uv run python scripts/prompts_version.py check + + # Byte-exact parity + deterministic (mock-model) semantic regression. + - name: Parity & semantic-regression tests + run: uv run python -m pytest tests/prompts -v diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index c9bafb44..97845242 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -7,13 +7,13 @@ cli: plugins: sources: - id: trunk - ref: v1.6.3 + ref: v1.10.2 uri: https://github.com/trunk-io/plugins # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - go@1.25.0 - - node@18.12.1 + - node@22.16.0 - python@3.10.8 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) lint: @@ -36,7 +36,7 @@ lint: - checkov@3.2.521 - git-diff-check - isort@8.0.1 - - markdownlint@0.48.0 + - markdownlint@0.49.0 - osv-scanner@2.3.5 - oxipng@10.1.0 - prettier@3.8.3 diff --git a/kai_mcp_solution_server/PROMPT_GOVERNANCE.md b/kai_mcp_solution_server/PROMPT_GOVERNANCE.md new file mode 100644 index 00000000..dfc4a42b --- /dev/null +++ b/kai_mcp_solution_server/PROMPT_GOVERNANCE.md @@ -0,0 +1,78 @@ +# Prompt Template Governance + +This document defines the change-management lifecycle for the LLM **prompt templates** the Kai +solution server uses to generate migration hints. It satisfies ISO/IEC 42001 Control **A.5.2 +(Governance and lifecycle of AI systems)**: the hint prompt is the instruction set defining the +server's hint-generation behavior, so it is treated as governed source code — version-controlled, +peer-reviewed, and validated by CI before distribution. + +## Scope — what is governed + +All model-bound prompt templates live as individual Jinja2 assets under +[`src/kai_mcp_solution_server/prompts/templates/`](src/kai_mcp_solution_server/prompts/templates/) and +are enumerated in +[`src/kai_mcp_solution_server/prompts/manifest.yaml`](src/kai_mcp_solution_server/prompts/manifest.yaml). +Today this is the live hint prompt, `generate_hint_v3`. Prompts are rendered by +[Jinja2](https://jinja.palletsprojects.com/) via `render_prompt(prompt_id, **context)` from +`kai_mcp_solution_server.prompts`; loops and structure live in the template, not in `server.py`. The +prompt-set is semantically versioned by `version:` in the manifest, kept in lockstep with the package +version in `pyproject.toml`. + +The `ast_diff_str` value (computed from `associate_files` + `extract_ast_info`) is data marshalling +and stays in Python; the template receives it as a variable. + +## Roles + +- **Prompt Engineer / Product Architect** (`@konveyor/kai-prompt-reviewers`) — required approver for any + change under `prompts/`. Owns prompt wording, the threat-model review, and the regression baseline. +- **Solution-server maintainers** — co-reviewers for the code that calls `render_prompt`. + +The prompts directory is assigned to the reviewers in [`.github/CODEOWNERS`](../.github/CODEOWNERS). + +## Lifecycle of a prompt change + +1. **Branch & edit** the template asset(s). Do not reintroduce prompt strings inline in `server.py`. +2. **Refresh checksums.** Run `python scripts/prompts_version.py update`. Bump `version:` in the + manifest (and `pyproject.toml`) if the change ships in a new release. +3. **Update parity expectations.** The byte-exact oracle (`tests/prompts/oracle.py`) is the historical + baseline. For an _intentional_ wording change, update the oracle and call it out in the PR. +4. **Complete the prompt-injection threat-model checklist** (below) in the PR. +5. **Open a PR** with DCO sign-off (`git commit -s`). CODEOWNERS routes it to a Prompt Engineer; the + [`prompt-validation`](../.github/workflows/prompt-validation.yml) workflow runs automatically. +6. **Merge** only after required review + green CI. + +## CI gates + +The `prompt-validation` workflow (triggered on `prompts/**`) runs `scripts/prompts_version.py check` +and `pytest tests/prompts`: + +- **Syntactic verification** — every template parses as Jinja2; declared variables exactly match the + template's referenced variables; no leftover `{var}` f-string interpolation. +- **Manifest/version governance** — every asset is declared (and vice-versa); content checksums match. +- **Byte-exact parity** — the template reproduces the pre-extraction prompt string exactly across the + incident / AST-diff matrix. +- **Semantic regression (deterministic, mock model)** — renders the prompt against a baseline of + migration scenarios and asserts the required scaffolding (output-format contract, each incident's + fields, the `AST Diff:` anchor) survives. + +## Prompt-injection threat-model checklist + +Complete for every template change that adds or alters an interpolation point: + +- [ ] **Untrusted interpolation** — `incident.message`, `incident.code_snip`, and `ast_diff_str` are + derived from analyzed user code. Confirm they remain data-only and cannot be read by the model + as instructions overriding the output-format contract. +- [ ] **Output-contract integrity** — do the `SUMMARY:` / `HINT:` anchors remain unambiguous so a + malicious code snippet can't spoof the expected response shape? +- [ ] **Rendering** — output is non-escaped by design (these are plain-text prompts); confirm no new + context (JSON, code fence) where unescaped interpolation enables injection. +- [ ] **Regression** — the semantic-regression baseline still passes. + +## Branch protection (repo admin action — not in code) + +A repository admin must, for `main` and `release-*`: + +1. **Require the status check** `Validate prompt templates`. +2. **Require review from Code Owners** so `prompts/**` edits need `@konveyor/kai-prompt-reviewers`. +3. **Create the `@konveyor/kai-prompt-reviewers` team** with the designated Prompt Engineers / + Product Architects. diff --git a/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md b/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md new file mode 100644 index 00000000..1ee04b98 --- /dev/null +++ b/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md @@ -0,0 +1,45 @@ +# ISO/IEC 42001 — Control A.5.2 Evidence Ledger (Kai Solution Server) + +**Control:** A.5.2 — Governance and lifecycle of AI systems +**Subject:** Prompt template governance, versioning, and automated validation for the Kai solution +server's hint-generation prompt. +**Status:** Implemented — the prompt is decoupled, version-controlled, peer-review mandated, and +CI-validated. + +This is the counterpart to the editor-extensions evidence ledger, which named `konveyor/kai` as the +owner of the solution-server prompt surface. This ledger lists the governed paths. + +## 1. Structured versioning & decoupling + +| Evidence | Path | +| ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| Dedicated prompt template directory (individual file assets) | [`src/kai_mcp_solution_server/prompts/templates/`](../src/kai_mcp_solution_server/prompts/templates/) | +| Semantic version + per-template manifest with checksums | [`src/kai_mcp_solution_server/prompts/manifest.yaml`](../src/kai_mcp_solution_server/prompts/manifest.yaml) | +| Render registry (template decoupled from app logic) | [`src/kai_mcp_solution_server/prompts/__init__.py`](../src/kai_mcp_solution_server/prompts/__init__.py) | + +The live hint prompt (`generate_hint_v3`) is rendered via `render_prompt(...)`; `server.py` no longer +embeds the prompt string. The dead `generate_hint_v1`/`v2` were removed in the same change. + +## 2. Automated CI validation pipeline + +| Evidence | Path | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Pipeline definition (triggered on `prompts/**`) | [`.github/workflows/prompt-validation.yml`](../../.github/workflows/prompt-validation.yml) | +| Syntactic verification + checksum/version governance | [`scripts/prompts_version.py`](../scripts/prompts_version.py) | +| Byte-exact parity + oracle baseline | [`tests/prompts/test_parity.py`](../tests/prompts/test_parity.py), [`tests/prompts/oracle.py`](../tests/prompts/oracle.py) | +| Semantic regression (deterministic, mock model, baseline dataset) | [`tests/prompts/test_semantic_regression.py`](../tests/prompts/test_semantic_regression.py) | + +## 3. Peer review & security policy + +| Evidence | Path | +| -------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Governance process, roles, lifecycle | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md) | +| Mandatory code-owner review for prompt assets | [`.github/CODEOWNERS`](../../.github/CODEOWNERS) | +| Prompt-injection threat-model checklist | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md#prompt-injection-threat-model-checklist) | +| Branch-protection requirements (repo-admin action) | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md#branch-protection-repo-admin-action--not-in-code) | + +## Outstanding (repo-admin / org action) + +- Create the `@konveyor/kai-prompt-reviewers` team (Prompt Engineers / Product Architects). +- Enable branch protection on `main` / `release-*` requiring the `Validate prompt templates` check and + Code-Owner review for the prompts directory. diff --git a/kai_mcp_solution_server/pyproject.toml b/kai_mcp_solution_server/pyproject.toml index 62b6bfcc..c303cd49 100644 --- a/kai_mcp_solution_server/pyproject.toml +++ b/kai_mcp_solution_server/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "tree-sitter>=0.24.0", "tree-sitter-java>=0.23.5", "types-pyyaml>=6.0.12.20250516", + "jinja2>=3.1.6", # Security floors for transitive dependencies (pinned to CVE-fixed releases). "aiohttp>=3.14.0", "python-dotenv>=1.2.2", diff --git a/kai_mcp_solution_server/requirements.txt b/kai_mcp_solution_server/requirements.txt index b9d76dc8..3955b587 100644 --- a/kai_mcp_solution_server/requirements.txt +++ b/kai_mcp_solution_server/requirements.txt @@ -72,6 +72,7 @@ jaraco-classes==3.4.0 jaraco-context==6.1.0 jaraco-functools==4.4.0 jeepney==0.9.0 +jinja2==3.1.6 jiter==0.12.0 jmespath==1.0.1 jsonpatch==1.33 diff --git a/kai_mcp_solution_server/scripts/prompts_version.py b/kai_mcp_solution_server/scripts/prompts_version.py new file mode 100644 index 00000000..75a9fd6b --- /dev/null +++ b/kai_mcp_solution_server/scripts/prompts_version.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Prompt template governance checks (ISO 42001 A.5.2). + +Validates the versioned prompt template set under +``src/kai_mcp_solution_server/prompts/``: + + - every ``.jinja`` template is declared in ``manifest.yaml`` (and vice-versa) + - content checksums in the manifest match the files on disk (drift detection) + - every template parses as Jinja2 (syntactic verification) + - declared variables exactly match the template's referenced variables, and no + leftover ``{var}`` f-string interpolation escaped into a template asset + +Usage: + python scripts/prompts_version.py check # CI gate (non-zero exit on problems) + python scripts/prompts_version.py update # recompute + write checksums +""" + +from __future__ import annotations + +import hashlib +import re +import sys +import tomllib +from pathlib import Path + +import yaml +from jinja2 import Environment, meta + +ROOT = Path(__file__).resolve().parent.parent +PROMPTS_DIR = ROOT / "src" / "kai_mcp_solution_server" / "prompts" +TEMPLATES_DIR = PROMPTS_DIR / "templates" +MANIFEST_PATH = PROMPTS_DIR / "manifest.yaml" +PYPROJECT_PATH = ROOT / "pyproject.toml" + +# Matches a leftover Python f-string placeholder like ``{incident.uri}`` while +# ignoring Jinja's own ``{{ ... }}`` / ``{% ... %}``. +_FSTRING_RE = re.compile(r"(? str: + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def load_manifest() -> dict: + return yaml.safe_load(MANIFEST_PATH.read_text()) + + +def check() -> int: + manifest = load_manifest() + templates = manifest.get("templates", []) + problems: list[str] = [] + env = Environment() # noqa: S701 # nosec B701 - parses templates, never renders + + # The prompt-set version must stay in lockstep with the package version so + # release metadata can't drift (see PROMPT_GOVERNANCE.md). + pkg_version = ( + tomllib.loads(PYPROJECT_PATH.read_text()).get("project", {}).get("version") + ) + if manifest.get("version") != pkg_version: + problems.append( + f"Prompt-set version ({manifest.get('version')}) does not match the package " + f"version in pyproject.toml ({pkg_version}). Keep them in lockstep." + ) + + on_disk = {p.relative_to(ROOT).as_posix() for p in TEMPLATES_DIR.rglob("*.jinja")} + declared = {t["path"] for t in templates} + for path in on_disk - declared: + problems.append(f"Template on disk is not declared in manifest.yaml: {path}") + for path in declared - on_disk: + problems.append( + f"Manifest declares a template that does not exist on disk: {path}" + ) + + for entry in templates: + abs_path = ROOT / entry["path"] + if not abs_path.exists(): + continue # missing-file already reported + src = abs_path.read_text() + + actual = sha256(src) + if entry.get("checksum") != actual: + problems.append( + f"Checksum drift for {entry['id']} ({entry['path']}). " + f"Manifest: {entry.get('checksum') or ''}, actual: {actual}. " + 'Run "python scripts/prompts_version.py update" and review the change.' + ) + + try: + ast = env.parse(src) + except Exception as err: # noqa: BLE001 - report any parse failure + problems.append( + f"Jinja2 parse error in {entry['id']} ({entry['path']}): {err}" + ) + continue + + if _FSTRING_RE.search(src): + problems.append( + f"Leftover {{...}} f-string interpolation found in {entry['id']} ({entry['path']})." + ) + + referenced = meta.find_undeclared_variables(ast) + declared_vars = set(entry.get("variables", [])) + for missing in declared_vars - referenced: + problems.append( + f'Declared variable "{missing}" is not referenced in {entry["id"]} ({entry["path"]}).' + ) + for undeclared in referenced - declared_vars: + problems.append( + f'Variable "{undeclared}" is used in {entry["id"]} ({entry["path"]}) ' + "but not declared in manifest.yaml." + ) + + if problems: + print("Prompt template validation FAILED:\n", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + print(f"\n{len(problems)} problem(s) found.", file=sys.stderr) + return 1 + + print( + f"Prompt template validation passed. version={manifest.get('version')}, " + f"templates={len(templates)}." + ) + return 0 + + +def update() -> int: + manifest = load_manifest() + for entry in manifest.get("templates", []): + entry["checksum"] = sha256((ROOT / entry["path"]).read_text()) + header = ( + "# Prompt template manifest — governed under ISO 42001 A.5.2.\n" + "# Checksums are maintained by scripts/prompts_version.py; do not edit by hand.\n" + ) + MANIFEST_PATH.write_text( + header + yaml.safe_dump(manifest, sort_keys=False, width=1000) + ) + print(f"Updated checksums for {len(manifest.get('templates', []))} template(s).") + return 0 + + +def main() -> int: + cmd = sys.argv[1] if len(sys.argv) > 1 else "" + if cmd == "check": + return check() + if cmd == "update": + return update() + print("Usage: python scripts/prompts_version.py ", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py new file mode 100644 index 00000000..e728cd83 --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py @@ -0,0 +1,42 @@ +"""Governed prompt templates for the solution server (ISO 42001 A.5.2). + +Prompts live as individual Jinja2 assets under ``templates/`` and are rendered +through :func:`render_prompt`. They are versioned and validated independently of +the application code; see ``manifest.yaml`` and ``PROMPT_GOVERNANCE.md``. +""" + +from __future__ import annotations + +from functools import lru_cache + +from jinja2 import Environment, PackageLoader, StrictUndefined, Template + +# One id per governed prompt -> its template filename. +_TEMPLATES: dict[str, str] = { + "generate_hint_v3": "generate_hint_v3.md.jinja", +} + +# autoescape is intentionally off: these are plain-text LLM prompts, not HTML, so +# HTML-escaping would corrupt code snippets in the rendered prompt (B701 N/A). +# keep_trailing_newline preserves the template's final newline verbatim; +# StrictUndefined makes a missing context variable fail loudly. +_env = Environment( # nosec B701 + loader=PackageLoader("kai_mcp_solution_server", "prompts/templates"), + autoescape=False, + keep_trailing_newline=True, + undefined=StrictUndefined, +) + + +@lru_cache(maxsize=None) +def _template(filename: str) -> Template: + return _env.get_template(filename) + + +def render_prompt(prompt_id: str, /, **context: object) -> str: + """Render a governed prompt template to its final string.""" + try: + filename = _TEMPLATES[prompt_id] + except KeyError: + raise KeyError(f"Unknown prompt id: {prompt_id!r}") from None + return _template(filename).render(**context) diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml new file mode 100644 index 00000000..975a6ecb --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml @@ -0,0 +1,12 @@ +# Prompt template manifest — governed under ISO 42001 A.5.2. +# Checksums are maintained by scripts/prompts_version.py; do not edit by hand. +version: 0.1.0 +engine: jinja2 +templates: + - id: generate_hint_v3 + path: src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja + governed: true + variables: + - incidents + - ast_diff_str + checksum: sha256:bd4dcf69a3525a0c914626551c521edaf326c4cb63e0fa7066d5c908a00a00aa diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja new file mode 100644 index 00000000..fbc0fa94 --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja @@ -0,0 +1,32 @@ +The following incidents had this accepted solution. Use the AST diffs below as a guiding pattern for migration. + +Generate a hint for the user so that they can migrate the code. + +IMPORTANT: Follow this EXACT output format: +--- +SUMMARY: +[concise summary of necessary changes] + +HINT: +[numbered steps with generic, reusable code examples] +--- + +Guidelines for high-quality response: +1. Keep SUMMARY concise and focused +2. Use numbered steps (1, 2, 3) in HINT section +3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...') +4. Write in direct, actionable tone +Incidents: +{% for incident in incidents -%} +Incident {{ loop.index }}: + URI: {{ incident.uri }} + Message: {{ incident.message }} + Code Snippet: {{ incident.code_snip }} + Line Number: {{ incident.line_number }} + Variables: {{ incident.variables }} + Violation: {{ incident.violation.ruleset_name }} - {{ incident.violation.violation_name }} + +{% endfor -%} +AST Diff: +{{ ast_diff_str }} + diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py b/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py index 21404a99..430638f0 100644 --- a/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py @@ -36,8 +36,8 @@ SolutionStatus, ViolationID, associate_files, - get_diff, ) +from kai_mcp_solution_server.prompts import render_prompt P = ParamSpec("P") T = TypeVar("T") @@ -568,158 +568,6 @@ async def tool_create_solution( ) -@with_db_recovery -async def generate_hint_v1( - kai_ctx: KaiSolutionServerContext, - client_id: str, -) -> None: - if kai_ctx.session_maker is None: - raise RuntimeError("Session maker not initialized") - async with kai_ctx.session_maker.begin() as session: - solutions_stmt = select(DBSolution).where( - DBSolution.client_id == client_id, - or_( - DBSolution.solution_status == SolutionStatus.ACCEPTED, - DBSolution.solution_status == SolutionStatus.MODIFIED, - ), - ) - solutions = (await session.execute(solutions_stmt)).scalars().all() - if len(solutions) == 0: - log( - f"No accepted or modified solutions found for client {client_id}. No hint generated." - ) - return - - for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Generate a hint for the user so that they can perform the same solution:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - - diff = get_diff( - [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], - [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], - ) - - prompt += "Solution:\n" f"{diff}\n\n" - - log(f"Generating hint for client {client_id} with prompt:\n{prompt}") - - if kai_ctx.model is None: - raise RuntimeError("Model not initialized") - response = await kai_ctx.model.ainvoke(prompt) - - log(f"Generated hint: {response.content}") - - hint = DBHint( - text=str(response.content), - violations=set( - incident.violation - for incident in solution.incidents - if incident.violation is not None - ), - solutions=set([solution]), - ) - session.add(hint) - - await session.flush() - - -@with_db_recovery -async def generate_hint_v2( - kai_ctx: KaiSolutionServerContext, - client_id: str, -) -> None: - # print(f"Generating hint for client {client_id}", file=sys.stderr) - if kai_ctx.session_maker is None: - raise RuntimeError("Session maker not initialized") - async with kai_ctx.session_maker.begin() as session: - solutions_stmt = select(DBSolution).where( - DBSolution.client_id == client_id, - or_( - DBSolution.solution_status == SolutionStatus.ACCEPTED, - DBSolution.solution_status == SolutionStatus.MODIFIED, - ), - ) - solutions = (await session.execute(solutions_stmt)).scalars().all() - if len(solutions) == 0: - print( - f"No accepted solutions found for client {client_id}. No hint generated.", - file=sys.stderr, - ) - return - - for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Generate a hint for the user so that they can create the same solution:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - - diff = associate_files( - [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], - [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], - ) - - ast_diffs: list[dict[str, Any]] = [] - for (_before_uri, _after_uri), (before_file, after_file) in diff.items(): - if before_file.content == after_file.content: - continue - - ast_diffs.append( - extract_ast_info(before_file.content, language=Language.JAVA).diff( - extract_ast_info(after_file.content, language=Language.JAVA) - ) - ) - - ast_diff_str = "\n\n".join(str(a) for a in ast_diffs if a is not None) - prompt += f"AST Diff:\n{ast_diff_str}\n\n" - - # print(f"Generating hint for client {client_id} with prompt:\n{prompt}", file=sys.stderr) - - if kai_ctx.model is None: - raise RuntimeError("Model not initialized") - response = await kai_ctx.model.ainvoke(prompt) - - # print(f"Generated hint: {response.content}", file=sys.stderr) - - hint = DBHint( - text=str(response.content), - violations=set( - incident.violation - for incident in solution.incidents - if incident.violation is not None - ), - solutions=set([solution]), - ) - session.add(hint) - - await session.flush() - - @with_db_recovery async def generate_hint_v3( kai_ctx: KaiSolutionServerContext, @@ -747,37 +595,6 @@ async def generate_hint_v3( return for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Use the AST diffs below as a guiding pattern for migration.\n\n" - "Generate a hint for the user so that they can migrate the code.\n\n" - "IMPORTANT: Follow this EXACT output format:\n" - "---\n" - "SUMMARY:\n" - "[concise summary of necessary changes]\n\n" - "HINT:\n" - "[numbered steps with generic, reusable code examples]\n" - "---\n\n" - "Guidelines for high-quality response:\n" - "1. Keep SUMMARY concise and focused\n" - "2. Use numbered steps (1, 2, 3) in HINT section\n" - "3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...')\n" - "4. Write in direct, actionable tone\n" - "Incidents:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - diff = associate_files( [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], @@ -795,7 +612,12 @@ async def generate_hint_v3( ) ast_diff_str = "\n\n".join(str(a) for a in ast_diffs if a is not None) - prompt += f"AST Diff:\n{ast_diff_str}\n\n" + + prompt = render_prompt( + "generate_hint_v3", + incidents=solution.incidents, + ast_diff_str=ast_diff_str, + ) if kai_ctx.model is None: raise RuntimeError("Model not initialized") diff --git a/kai_mcp_solution_server/tests/prompts/__init__.py b/kai_mcp_solution_server/tests/prompts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kai_mcp_solution_server/tests/prompts/oracle.py b/kai_mcp_solution_server/tests/prompts/oracle.py new file mode 100644 index 00000000..869623e1 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/oracle.py @@ -0,0 +1,49 @@ +"""Parity oracle — the byte-exact baseline for the prompt-template migration. + +This is a VERBATIM copy of the ``generate_hint_v3`` prompt-building logic as it +existed inline in ``server.py`` immediately before extraction. The parity suite +asserts that ``render_prompt("generate_hint_v3", ...)`` reproduces this string +byte-for-byte. Do not "clean up" anything here — quirks (the ``- `` triple +space before the violation name, the trailing blank line) are intentional and +are exactly what the template must reproduce. +""" + +from __future__ import annotations + +from typing import Any + + +def oracle_generate_hint_v3(incidents: list[Any], ast_diff_str: str) -> str: + prompt = ( + "The following incidents had this accepted solution. " + "Use the AST diffs below as a guiding pattern for migration.\n\n" + "Generate a hint for the user so that they can migrate the code.\n\n" + "IMPORTANT: Follow this EXACT output format:\n" + "---\n" + "SUMMARY:\n" + "[concise summary of necessary changes]\n\n" + "HINT:\n" + "[numbered steps with generic, reusable code examples]\n" + "---\n\n" + "Guidelines for high-quality response:\n" + "1. Keep SUMMARY concise and focused\n" + "2. Use numbered steps (1, 2, 3) in HINT section\n" + "3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...')\n" + "4. Write in direct, actionable tone\n" + "Incidents:\n" + ) + + for i, incident in enumerate(incidents): + prompt += ( + f"Incident {i + 1}:\n" + f" URI: {incident.uri}\n" + f" Message: {incident.message}\n" + f" Code Snippet: {incident.code_snip}\n" + f" Line Number: {incident.line_number}\n" + f" Variables: {incident.variables}\n" + f" Violation: {incident.violation.ruleset_name} - " + f" {incident.violation.violation_name}\n\n" + ) + + prompt += f"AST Diff:\n{ast_diff_str}\n\n" + return prompt diff --git a/kai_mcp_solution_server/tests/prompts/test_parity.py b/kai_mcp_solution_server/tests/prompts/test_parity.py new file mode 100644 index 00000000..02c40d85 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/test_parity.py @@ -0,0 +1,75 @@ +"""Byte-exact parity: the Jinja template must reproduce the original prompt +string exactly, across the full incident/ast-diff matrix.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from kai_mcp_solution_server.prompts import render_prompt + +from .oracle import oracle_generate_hint_v3 + + +def _incident( + uri: str = "file:///src/Foo.java", + message: str = "javax.* must become jakarta.*", + code_snip: str = "import javax.persistence.Entity;", + line_number: int = 12, + variables: object = None, + ruleset_name: str = "javax-to-jakarta", + violation_name: str = "javax-import", +) -> SimpleNamespace: + if variables is None: + variables = {"kind": "import"} + return SimpleNamespace( + uri=uri, + message=message, + code_snip=code_snip, + line_number=line_number, + variables=variables, + violation=SimpleNamespace( + ruleset_name=ruleset_name, violation_name=violation_name + ), + ) + + +_AST_DIFF = ( + "class Foo {\n- @javax.persistence.Entity\n+ @jakarta.persistence.Entity\n}" +) + +_CASES = { + "0 incidents, empty ast": ([], ""), + "1 incident, empty ast": ([_incident()], ""), + "1 incident, populated ast": ([_incident()], _AST_DIFF), + "N incidents, populated ast": ( + [ + _incident(), + _incident( + uri="file:///src/Bar.java", + message="multi\nline\nmessage", + code_snip="line1\nline2", + line_number=99, + variables={"a": 1, "b": ["x", "y"]}, + violation_name="ejb-usage", + ), + _incident(message="special <>&\"' chars", variables=None), + ], + _AST_DIFF, + ), +} + + +@pytest.mark.parametrize("name", list(_CASES)) +def test_v3_renders_byte_identical_to_oracle(name: str) -> None: + incidents, ast_diff_str = _CASES[name] + rendered = render_prompt( + "generate_hint_v3", incidents=incidents, ast_diff_str=ast_diff_str + ) + assert rendered == oracle_generate_hint_v3(incidents, ast_diff_str) + + +def test_unknown_prompt_id_raises() -> None: + with pytest.raises(KeyError): + render_prompt("does-not-exist") diff --git a/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py new file mode 100644 index 00000000..dabc1310 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py @@ -0,0 +1,76 @@ +"""Deterministic (mock-model) semantic regression. + +Renders the hint prompt against a baseline of migration scenarios and asserts the +scaffolding a model needs to produce a usable hint survives any wording change: +the output-format contract, every incident's identifying fields, and the AST diff. +No live model is involved. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from kai_mcp_solution_server.prompts import render_prompt + +_OUTPUT_CONTRACT = [ + "IMPORTANT: Follow this EXACT output format:", + "SUMMARY:", + "HINT:", + "Incidents:", + "AST Diff:", +] + + +def _incident(uri: str, message: str, violation_name: str) -> SimpleNamespace: + return SimpleNamespace( + uri=uri, + message=message, + code_snip="import javax.persistence.Entity;", + line_number=1, + variables={"kind": "import"}, + violation=SimpleNamespace( + ruleset_name="javax-to-jakarta", violation_name=violation_name + ), + ) + + +_BASELINE = { + "javaee->quarkus": ( + [ + _incident( + "file:///Foo.java", + "Replace javax.persistence with jakarta.persistence", + "javax-import", + ) + ], + "class Foo {\n- @javax.persistence.Entity\n+ @jakarta.persistence.Entity\n}", + ), + "multi-incident": ( + [ + _incident("file:///A.java", "Remove EJB usage", "ejb-usage"), + _incident("file:///B.java", "Migrate persistence.xml", "persistence-xml"), + ], + "class A {}\n\nclass B {}", + ), +} + + +@pytest.mark.parametrize("name", list(_BASELINE)) +def test_v3_preserves_migration_scaffolding(name: str) -> None: + incidents, ast_diff_str = _BASELINE[name] + rendered = render_prompt( + "generate_hint_v3", incidents=incidents, ast_diff_str=ast_diff_str + ) + + for anchor in _OUTPUT_CONTRACT: + assert anchor in rendered, f"missing output-contract anchor: {anchor!r}" + for incident in incidents: + assert incident.uri in rendered + assert incident.message in rendered + assert incident.code_snip in rendered + assert str(incident.line_number) in rendered + assert str(incident.variables) in rendered + assert incident.violation.violation_name in rendered + assert ast_diff_str in rendered diff --git a/kai_mcp_solution_server/uv.lock b/kai_mcp_solution_server/uv.lock index 0311fa34..aa251620 100644 --- a/kai_mcp_solution_server/uv.lock +++ b/kai_mcp_solution_server/uv.lock @@ -1626,6 +1626,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.12.0" @@ -1793,6 +1805,7 @@ dependencies = [ { name = "alembic" }, { name = "asyncpg" }, { name = "fastmcp" }, + { name = "jinja2" }, { name = "langchain" }, { name = "langchain-aws" }, { name = "langchain-azure-ai" }, @@ -1840,6 +1853,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.15.2" }, { name = "asyncpg", specifier = ">=0.30.0" }, { name = "fastmcp", specifier = ">=2.8.0,<3" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "langchain", specifier = ">=1.2.4" }, { name = "langchain-aws", specifier = ">=1.2.1" }, { name = "langchain-azure-ai", specifier = ">=1.0.0" },