From 56a30f587b39fad12e2430b3cd1a98faa6b12eee Mon Sep 17 00:00:00 2001 From: Gabor Bakos Date: Fri, 12 Jun 2026 09:54:36 +0200 Subject: [PATCH 1/2] refactor(deep-scan): remove the drift step, salvage health + baseline into Step 9 Finalize The drift step (old Step 9) was the slowest post-intent-layer phase and its findings bypassed the verified channel: no triggering_call_site requirement, no backward verification, no hysteresis, regenerated from scratch each run. Everything it claimed to cover is owned by machinery that does it better: - Decision violations / pitfall triggers / trade-off undermining / schema drift: the Wave 2 Risk agent's invariant walk (Step 5b) + edit-time hooks. - Recency coverage: new recency sweep in the Risk agent - in incremental runs the changed_files list (already injected into the Wave 2 preamble) is read file-by-file against documented invariants and per-folder CLAUDE.md patterns, and the resulting findings flow through the verifier. - Recurrence tracking: findings.json (stable ids, confirmed_in_scan, hysteresis) replaces the LLM prose-diff of scan_report.md. Salvaged into the new Step 9 (Finalize, plain bash, no subagent): health measurement (health.json + history), complete-step 9 + save-baseline (the --incremental dependency), telemetry flush, and the closing summary. Removed: drift.py (722 lines), step-9-drift.md, step-10-telemetry.md (absorbed into step-9-finalize.md), templates/scan-report.md, the extract_output deep-drift/recent-files subcommands, the dead filter-ignored subcommand, and the scan_report/drift_report share-bundle sources. Step 8 cleanup and the pip install path now delete stale drift artifacts on upgrade. Accepted losses (signed off): semantic near-twin duplication detection has no successor (health.json verbosity = exact clones only); the share viewer loses the prose report page; telemetry trend continuity breaks on the drift step key. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 4 +- README.md | 6 +- archie/assets/workflow/deep-scan/SKILL.md | 9 +- .../fragments/telemetry-conventions.md | 2 +- .../deep-scan/steps/step-10-telemetry.md | 22 - .../deep-scan/steps/step-5-wave2-reasoning.md | 2 + .../workflow/deep-scan/steps/step-5b-risk.md | 2 + .../deep-scan/steps/step-7-intent-layer.md | 4 +- .../deep-scan/steps/step-8-cleanup.md | 6 + .../workflow/deep-scan/steps/step-9-drift.md | 198 ----- .../deep-scan/steps/step-9-finalize.md | 70 ++ .../deep-scan/templates/scan-report.md | 74 -- archie/install.py | 12 +- archie/manifest_data.py | 11 +- archie/standalone/_common.py | 2 +- archie/standalone/drift.py | 722 ------------------ archie/standalone/extract_output.py | 80 +- archie/standalone/intent_layer.py | 24 +- archie/standalone/sync.py | 2 +- archie/standalone/telemetry.py | 2 +- archie/standalone/upload.py | 70 +- docs/ARCHITECTURE.md | 72 +- npm-package/README.md | 2 +- npm-package/assets/_common.py | 2 +- npm-package/assets/_install_pkg/install.py | 12 +- .../assets/_install_pkg/manifest_data.py | 11 +- npm-package/assets/drift.py | 722 ------------------ npm-package/assets/extract_output.py | 80 +- npm-package/assets/intent_layer.py | 24 +- npm-package/assets/sync.py | 2 +- npm-package/assets/telemetry.py | 2 +- npm-package/assets/upload.py | 70 +- .../assets/workflow/deep-scan/SKILL.md | 9 +- .../fragments/telemetry-conventions.md | 2 +- .../deep-scan/steps/step-10-telemetry.md | 22 - .../deep-scan/steps/step-5-wave2-reasoning.md | 2 + .../workflow/deep-scan/steps/step-5b-risk.md | 2 + .../deep-scan/steps/step-7-intent-layer.md | 4 +- .../deep-scan/steps/step-8-cleanup.md | 6 + .../workflow/deep-scan/steps/step-9-drift.md | 198 ----- .../deep-scan/steps/step-9-finalize.md | 70 ++ .../deep-scan/templates/scan-report.md | 74 -- npm-package/bin/archie.mjs | 2 +- tests/test_comprehensive_mode_integration.py | 29 +- tests/test_comprehensive_wiring.py | 26 +- tests/test_ignore_patterns.py | 14 +- tests/test_telemetry_agents.py | 2 +- tests/test_upload.py | 7 +- 48 files changed, 333 insertions(+), 2459 deletions(-) delete mode 100644 archie/assets/workflow/deep-scan/steps/step-10-telemetry.md delete mode 100644 archie/assets/workflow/deep-scan/steps/step-9-drift.md create mode 100644 archie/assets/workflow/deep-scan/steps/step-9-finalize.md delete mode 100644 archie/assets/workflow/deep-scan/templates/scan-report.md delete mode 100644 archie/standalone/drift.py delete mode 100644 npm-package/assets/drift.py delete mode 100644 npm-package/assets/workflow/deep-scan/steps/step-10-telemetry.md delete mode 100644 npm-package/assets/workflow/deep-scan/steps/step-9-drift.md create mode 100644 npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md delete mode 100644 npm-package/assets/workflow/deep-scan/templates/scan-report.md diff --git a/CLAUDE.md b/CLAUDE.md index a25d0e76..1dd7e524 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,9 @@ See `archie/benchmark/README.md`. 5. **Render** — Deterministic JSON→Markdown (CLAUDE.md, AGENTS.md, rule files) 6. **Validate** — Cross-reference output against actual codebase 7. **Intent Layer** — AI-generated per-folder CLAUDE.md via bottom-up DAG -8. **Scan report** — Phase 4 of Step 9 writes `.archie/scan_report.md` with ranked findings (so `/archie-share` and future trend runs pick it up) +8. **Finalize** — Health metrics (`health.json` + history), incremental baseline marker, telemetry flush, closing summary + +Findings live in `.archie/findings.json` (compounding store: stable ids, verifier + hysteresis in Step 5); `/archie-share` ships them from there. In incremental mode the Risk agent additionally sweeps the changed files against documented invariants and per-folder CLAUDE.md patterns (the recency sweep). ## Key Data Model diff --git a/README.md b/README.md index aed52bd1..5f449753 100644 --- a/README.md +++ b/README.md @@ -67,16 +67,14 @@ Run `/archie-deep-scan` once for the baseline, then `/archie-deep-scan --increme Health: Erosion 0.95 (front) / 0.69 (back) · Gini 0.92 / 0.76 · LoC 76,903 / 45,261 - Drift: 14 errors (raw ipcRenderer exposure, API key in logs, duplicate startup + Findings: 14 errors (raw ipcRenderer exposure, API key in logs, duplicate startup handlers, stale WebSocket closures, DI-layer imports from utils, …) 14 warnings (sync I/O in async paths, monolith routers, compat-hook CRUD leak, …) + — each verified against its triggering call site before it ships Top risks: IPC security hole · API key exposure · stale WebSocket state · circular deps & layer violations · duplicate startup handler - Semantic duplication: 3 groups (placeholder_resolver clone, dual WebSocketMappingService, - sidebar state in two contexts) - Archie is now active. Rules will be enforced on every code change. Run /archie-deep-scan --incremental after code changes to refresh the analysis. ``` diff --git a/archie/assets/workflow/deep-scan/SKILL.md b/archie/assets/workflow/deep-scan/SKILL.md index 3bd31f51..9551fd8c 100644 --- a/archie/assets/workflow/deep-scan/SKILL.md +++ b/archie/assets/workflow/deep-scan/SKILL.md @@ -1,6 +1,6 @@ --- name: archie-deep-scan -description: Comprehensive architecture baseline scan (15-20 min). Two-wave AI analysis producing blueprint.json, per-folder CLAUDE.md, AI-synthesized rules, health metrics, and drift detection. Use for first-time baselines or major refactors. +description: Comprehensive architecture baseline scan. Two-wave AI analysis producing blueprint.json, per-folder CLAUDE.md, AI-synthesized rules, and health metrics. Use for first-time baselines or major refactors. --- # Archie Deep Scan — Comprehensive Architecture Baseline @@ -55,7 +55,7 @@ Check the user's message (ARGUMENTS) for flags: > > **Treat every item-count anywhere in this workflow — `N-M`, "up to N", "top N", "the N most", "soft floor of N", "a handful", "the most important" — as a FLOOR with no ceiling.** Produce every item that genuinely meets its quality bar; never trim to a number, never stop at a "natural" count. The quality bar is unchanged: do NOT pad, invent, or weaken per-item rigor to inflate counts. > -> Only two limits survive comprehensive mode: **(1)** the architecture diagram stays **8-12 nodes** (readability), and **(2)** the scripts' mechanical safety/context budgets (per-file read size, per-batch token budget, recursion). Everything else — rules, findings, pitfalls, decisions, trade-offs, components, guidelines, examples, drift findings, naming examples, per-field prose length — is uncapped. +> Only two limits survive comprehensive mode: **(1)** the architecture diagram stays **8-12 nodes** (readability), and **(2)** the scripts' mechanical safety/context budgets (per-file read size, per-batch token budget, recursion). Everything else — rules, findings, pitfalls, decisions, trade-offs, components, guidelines, examples, naming examples, per-field prose length — is uncapped. > > Whenever a step dispatches a sub-agent in comprehensive depth, prepend the contract line shown at that dispatch site so the sub-agent inherits this rule. @@ -129,7 +129,7 @@ STATUS=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state | 6 | AI rule synthesis | | 7 | Intent Layer | | 8 | Cleanup | - | 9 | Drift detection | + | 9 | Finalize (health + telemetry) | Ask the user how to proceed — {{>ask_user}}: - **question:** (build dynamically) `"A previous deep-scan stopped after Step {LAST} ({step_name})."` — and if `ENRICH_DONE > 0`, append `" The Intent Layer got {ENRICH_DONE} folders in before stopping."` — then `"What do you want to do?"` @@ -191,7 +191,6 @@ If `START_STEP > N` (the Preamble decided to skip earlier steps), do not Read or | 6 | AI rule synthesis | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-6-rule-synthesis.md` | | 7 | Intent Layer — per-folder CLAUDE.md | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-7-intent-layer.md` | | 8 | Cleanup | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-8-cleanup.md` | -| 9 | Drift detection & architectural assessment | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-9-drift.md` | -| 10 | Final telemetry flush | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-10-telemetry.md` | +| 9 | Finalize — health metrics, baseline, telemetry flush | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-9-finalize.md` | Step 3's `orchestration.md` in turn references four sub-agent prompt files plus a shared `grounding-rules.md` (all under `{{WORKFLOW_ROOT}}/deep-scan/steps/step-3-wave1/`) — read those as the orchestration instructs. diff --git a/archie/assets/workflow/deep-scan/fragments/telemetry-conventions.md b/archie/assets/workflow/deep-scan/fragments/telemetry-conventions.md index 58e08624..67ca3120 100644 --- a/archie/assets/workflow/deep-scan/fragments/telemetry-conventions.md +++ b/archie/assets/workflow/deep-scan/fragments/telemetry-conventions.md @@ -1,6 +1,6 @@ ## Telemetry conventions -Every Step 1–9 records its start timestamp to disk via `telemetry.py mark` as its first action. The mark auto-closes the previous step's `completed_at`, mirroring the "next step's start = prior step's end" convention. Step 9 finishes its own completion with `telemetry.py finish`. After Step 10, `telemetry.py write` consumes the persisted `.archie/telemetry/_current_run.json` and emits the final `deep-scan_.json`. +Every Step 1–9 records its start timestamp to disk via `telemetry.py mark` as its first action. The mark auto-closes the previous step's `completed_at`, mirroring the "next step's start = prior step's end" convention. Step 9 finishes its own completion with `telemetry.py finish`, then `telemetry.py write` consumes the persisted `.archie/telemetry/_current_run.json` and emits the final `deep-scan_.json`. Shell-variable fallback: the existing `TELEMETRY_STEPN_START` shell variables are still set for readability, but they are **not load-bearing** — the disk file is the source of truth. This makes the pipeline safe to `/compact` mid-run without losing timing data. diff --git a/archie/assets/workflow/deep-scan/steps/step-10-telemetry.md b/archie/assets/workflow/deep-scan/steps/step-10-telemetry.md deleted file mode 100644 index 98d2be13..00000000 --- a/archie/assets/workflow/deep-scan/steps/step-10-telemetry.md +++ /dev/null @@ -1,22 +0,0 @@ -## Step 10: Write telemetry - -Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. - -If the Intent Layer was skipped (INTENT_LAYER=no), mark it so explicitly: - -```bash -if [ "$INTENT_LAYER" = "no" ]; then - python3 .archie/telemetry.py extra "$PROJECT_ROOT" intent_layer skipped=true -fi -``` - -Then flush the in-flight file into the final `.archie/telemetry/deep-scan_.json`: - -```bash -python3 .archie/telemetry.py finish "$PROJECT_ROOT" -python3 .archie/telemetry.py write "$PROJECT_ROOT" -``` - -`write` auto-closes any still-open step with `now`, emits the final timestamped JSON, then deletes `_current_run.json` so the next deep-scan starts fresh. If telemetry fails for any reason, do not abort — telemetry is informational only. - -**Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. diff --git a/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md b/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md index 280a76c5..3984a591 100644 --- a/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md +++ b/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md @@ -71,6 +71,8 @@ Spawn the **Product** sub-agent only when `DOMAIN_LAW_COUNT` is greater than 0. - **If SCAN_MODE = "incremental":** > INCREMENTAL UPDATE. The architecture was previously analyzed — `$PROJECT_ROOT/.archie/blueprint.json` is the current full architecture and `$PROJECT_ROOT/.archie/blueprint_raw.json` carries the structural changes from Step 4. These files changed: [list `changed_files`]. Update ONLY the sections you own that are affected by these changes, and return ONLY what changed — unchanged sections are preserved by the patch merge. Use the 4-field contract (`problem_statement`, `evidence`, `root_cause`, `fix_direction`) when writing finding or pitfall entries. + The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. + **Output contract — append to each prompt, substituting that sub-agent's output path from the table as the "file path named above":** ``` diff --git a/archie/assets/workflow/deep-scan/steps/step-5b-risk.md b/archie/assets/workflow/deep-scan/steps/step-5b-risk.md index a933ec21..b696dddf 100644 --- a/archie/assets/workflow/deep-scan/steps/step-5b-risk.md +++ b/archie/assets/workflow/deep-scan/steps/step-5b-risk.md @@ -31,6 +31,8 @@ The `f_0001` shape we are guarding against: AI sees an AGENTS.md mandate, finds **APPROACH — anchor synthesis to documented invariants.** Instead of speculatively asking *"what could go wrong?"*, walk the documented invariants in AGENTS.md, root `CLAUDE.md`, per-folder `CLAUDE.md` (Anti-Patterns and Patterns sections — `.archie/maintainer_guardrails.json` if available), and `blueprint.pitfalls`. For each invariant, ask: *"is there code in this corpus that violates it? Quote it verbatim."* If yes ⇒ that quote is the `triggering_call_site` of a finding. If the invariant is real but uniformly enforced ⇒ no finding (and no need to re-emit a pitfall already in the store). This adversarial framing converts the loose "find problems" task into a falsifiable evidence-gathering pass. +**Recency sweep (only when the mode preamble carries a "These files changed:" list — incremental runs).** That list is your sweep scope: Read each listed file (skip ones that no longer exist) plus its folder's `CLAUDE.md` and parent folder's `CLAUDE.md` when they exist. Check each file against the documented invariants exactly as above — the per-folder Patterns/Anti-Patterns are first-class invariants here, alongside `blueprint.json` decisions, trade-offs (`violation_signals`), and pitfalls (`stems_from`). A violation with a verbatim caller becomes a normal finding (full 4-field shape + `triggering_call_site`); folder-pattern erosion with no firing call site is a pitfall (upgrade the existing one if the class is already tracked). The list is bounded by definition — read every file on it; do not sample. + **Primary goal — emit NEW findings.** You have the overall picture (all Wave 1 output plus source files). Your highest-leverage work is surfacing problems that are NOT already in findings.json — things only visible from the whole-system view: cross-component coupling, pattern breakdowns that individual agents miss, constraint violations implied by the decision chain, gaps between what the blueprint claims and what the code does. Spend the bulk of your cognitive budget here. For each new finding: next-free `f_NNNN` id, `first_seen` = today, `confirmed_in_scan` = 1, `depth: "canonical"`, `source: "deep:synthesis"`, AND a non-empty `triggering_call_site`. **Novelty check before emitting.** Before you add a "new" finding, verify it is genuinely new: scan the existing store for any entry with overlapping `problem_statement` meaning OR overlapping `applies_to` files. If the same problem is already tracked under a different wording, DO NOT mint a new id — instead upgrade the existing entry (see below). A new finding must describe something the store doesn't already cover. diff --git a/archie/assets/workflow/deep-scan/steps/step-7-intent-layer.md b/archie/assets/workflow/deep-scan/steps/step-7-intent-layer.md index 67e499b9..35752b1e 100644 --- a/archie/assets/workflow/deep-scan/steps/step-7-intent-layer.md +++ b/archie/assets/workflow/deep-scan/steps/step-7-intent-layer.md @@ -9,7 +9,7 @@ TELEMETRY_STEP7_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") **If START_STEP > 7, skip this step.** -**If `INTENT_LAYER=no` (user opted out in Step E), skip this entire step.** Print a one-line note to the user: *"Intent Layer skipped (no per-folder CLAUDE.md generated). Root CLAUDE.md + rule files still written. You can run `{{COMMAND_PREFIX}}archie-intent-layer` later if you change your mind."* Then proceed to Step 8. The `intent_layer` telemetry step will record zero elapsed time (its `started_at == completed_at`) and carry `"skipped": true` (see Step 10). +**If `INTENT_LAYER=no` (user opted out in Step E), skip this entire step.** Print a one-line note to the user: *"Intent Layer skipped (no per-folder CLAUDE.md generated). Root CLAUDE.md + rule files still written. You can run `{{COMMAND_PREFIX}}archie-intent-layer` later if you change your mind."* Then proceed to Step 8. The `intent_layer` telemetry step will record zero elapsed time (its `started_at == completed_at`) and carry `"skipped": true` (see Step 9). **If `INTENT_LAYER=yes`, execute this step fully. Do NOT ask the user whether to run, skip, or reduce scope. Do NOT offer alternatives. Run all batches as instructed below.** @@ -65,7 +65,7 @@ Then execute Phases 1–4 from that file, using `PROJECT_ROOT` in place of `$PWD ### ✓ Compact Checkpoint C — after Intent Layer -Only meaningful when `INTENT_LAYER=yes`. Step 7 has just pushed dozens-to-hundreds of {{ANALYSIS_MODEL}} subagent transcripts into conversation context; those are now fully persisted to `.archie/enrichments/*.json` and merged into per-folder `CLAUDE.md` files. Compacting here gives Step 9 (Drift Assessment) a fresh context, which matters because drift assessment reads blueprint + drift_report + CLAUDE.md files and benefits from focused attention. +Only meaningful when `INTENT_LAYER=yes`. Step 7 has just pushed dozens-to-hundreds of {{ANALYSIS_MODEL}} subagent transcripts into conversation context; those are now fully persisted to `.archie/enrichments/*.json` and merged into per-folder `CLAUDE.md` files. Compacting here gives the remaining bookkeeping steps (Cleanup, Finalize) a fresh context, so the run finishes reliably even after a very large Intent Layer pass. If `INTENT_LAYER=no` (opted out in Step E), skip this checkpoint — Checkpoint A already covered it. diff --git a/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md b/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md index 0c01a748..fe4e15b5 100644 --- a/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md +++ b/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md @@ -12,6 +12,12 @@ TELEMETRY_STEP8_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") rm -f .archie/tmp/archie_sub*_$PROJECT_NAME.json .archie/tmp/archie_rules_$PROJECT_NAME.json .archie/tmp/archie_intent_prompt_${PROJECT_NAME}_*.txt .archie/tmp/archie_enrichment_${PROJECT_NAME}_*.json ``` +Remove artifacts from the retired drift step (projects upgraded from older Archie versions may still carry them; nothing reads these anymore): + +```bash +rm -f .archie/drift_report.json .archie/scan_report.md +``` + ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 8 ``` diff --git a/archie/assets/workflow/deep-scan/steps/step-9-drift.md b/archie/assets/workflow/deep-scan/steps/step-9-drift.md deleted file mode 100644 index e2565969..00000000 --- a/archie/assets/workflow/deep-scan/steps/step-9-drift.md +++ /dev/null @@ -1,198 +0,0 @@ -## Step 9: Drift Detection & Architectural Assessment - -**Telemetry:** -```bash -python3 .archie/telemetry.py mark "$PROJECT_ROOT" deep-scan drift -TELEMETRY_STEP9_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -``` - -**If START_STEP > 9, skip this step.** - -### Phase 0: Health measurement - -```bash -python3 .archie/measure_health.py "$PROJECT_ROOT" > "$PROJECT_ROOT/.archie/health.json" 2>/dev/null -``` - -Save health scores to history for trending: - -```bash -python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep -``` - -### Phase 1: Mechanical drift scan - -```bash -python3 .archie/drift.py "$PROJECT_ROOT" -``` - -### Phase 2: Deep architectural drift (AI) - -Set the drift window by depth. Default depth: last 30 days, capped at 100 files. When `DEPTH=comprehensive`: full history, effectively unbounded. The pipeline shape stays identical in both depths — only these two vars change. Use a bash **array** for the `--since` argument so the value (which contains a space) is passed as a single argument, not word-split. - -```bash -# Re-derive DEPTH from persisted state so this late step does not depend on the -# shell variable surviving from Phase 0 across a long run / compaction. -DEPTH=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state.json --query .run_context.depth 2>/dev/null) -if [ "$DEPTH" = "comprehensive" ]; then - SINCE_ARGS=() # full history - DRIFT_MAX=1000000 # effectively unbounded -else - SINCE_ARGS=(--since="30 days ago") - DRIFT_MAX=100 -fi -``` - -Identify files to analyze: -```bash -git -C "$PROJECT_ROOT" log --name-only --pretty=format: ${SINCE_ARGS[@]+"${SINCE_ARGS[@]}"} -- '*.kt' '*.java' '*.swift' '*.ts' '*.tsx' '*.py' '*.go' '*.rs' \ - | sort -u | python3 .archie/intent_layer.py filter-ignored "$PROJECT_ROOT" | head -n "$DRIFT_MAX" -``` -If that returns nothing (new repo or no recent changes), use all source files from the scan: -```bash -python3 .archie/extract_output.py recent-files "$PROJECT_ROOT/.archie/scan.json" -``` - -For each file (batch into groups of ~15), collect: -- The file's content -- Its folder's CLAUDE.md **if it exists** (per-folder patterns, anti-patterns — these were generated in Step 7, but may be missing if Step 7 was skipped or partially completed) -- Its parent folder's CLAUDE.md **if it exists** - -Read `$PROJECT_ROOT/.archie/blueprint.json` — specifically `decisions.key_decisions`, `decisions.decision_chain`, `decisions.trade_offs` (with `violation_signals`), `pitfalls` (with `stems_from`), `communication.patterns`, `development_rules`. - -Read `$PROJECT_ROOT/.archie/drift_report.json` (mechanical findings from Phase 1). - -Spawn a **{{ANALYSIS_MODEL}} subagent** with the file contents, their folder CLAUDE.md files, and the blueprint context. {{>dispatch_single}} Tell it: - -> You are an architecture reviewer. You have the project's architectural blueprint (decisions, trade-offs, pitfalls, patterns), per-folder CLAUDE.md files describing expected patterns, mechanical drift findings (already detected), and source files to review. -> -> Find **deep architectural violations** — problems that pattern matching cannot catch. For each finding, return: -> - `folder`: the folder path -> - `file`: the specific file -> - `type`: one of `decision_violation`, `pattern_erosion`, `trade_off_undermined`, `pitfall_triggered`, `responsibility_leak`, `abstraction_bypass`, `semantic_duplication` -> - `severity`: `error` or `warn` -> - `decision_or_pattern`: which architectural decision, pattern, or pitfall this violates (reference by name from the blueprint) -> - `evidence`: the specific code (function name, class, line pattern) that demonstrates the violation -> - `message`: one sentence explaining what's wrong and why it matters -> -> Focus on: -> 1. **Decision violations** — code that contradicts a key architectural decision -> 2. **Pattern erosion** — code that doesn't follow the patterns described in its folder's CLAUDE.md -> 3. **Trade-off undermining** — code that works against an accepted trade-off (check `violation_signals`) -> 4. **Pitfall triggers** — code that falls into a documented pitfall (check `stems_from` chains) -> 5. **Responsibility leaks** — a component doing work that belongs to another component -> 6. **Abstraction bypass** — code reaching through a layer instead of using the intended interface -> 7. **Semantic duplication** — functions/methods with different signatures but essentially the same logic. AI agents frequently copy-paste a function, tweak the name/parameters, and leave the body identical or near-identical. Look for: functions with similar names (e.g., `getText`/`getTexts`, `loadUser`/`fetchUser`), functions in different files that do the same thing with slightly different types, helper functions reimplemented instead of shared. For each, use type `semantic_duplication` and explain what's duplicated and which function should be the canonical one. -> 8. **Schema drift** — when `blueprint.data_models` is non-empty, give it elevated attention. Touched files inside any `data_models[*].location` or `persistence_stores[*].migrations_dir` are signal-rich and merit extra scrutiny: did a migration ship without the corresponding ORM-model update (or vice versa)? Did a new column appear in code without a migration? Does a write path bypass the documented `lifecycle.how_to_read` repository? Use type `decision_violation` (when the change contradicts a documented invariant in `data_models[*].invariants`) or `pattern_erosion` (when it contradicts `data_models[*].lifecycle`). -> -> Do NOT report: style/formatting/naming (the script handles those), generic best-practice violations not grounded in THIS project's blueprint, or issues already in the mechanical drift report. -> -> Return JSON: `{"deep_findings": [...]}` - -Instruct the reviewer subagent to write its own output (append to its prompt). The "file path named above" is `.archie/tmp/archie_deep_drift.json`: - -``` ---- -OUTPUT CONTRACT (mandatory): -{{>output_contract}} -``` - -After the agent's confirmation returns, extract and clean up: - -```bash -python3 .archie/extract_output.py deep-drift .archie/tmp/archie_deep_drift.json "$PROJECT_ROOT/.archie/drift_report.json" -rm -f .archie/tmp/archie_deep_drift.json -``` - -### Phase 3: Present the combined assessment - -Read `$PROJECT_ROOT/.archie/blueprint.json` and `$PROJECT_ROOT/.archie/drift_report.json` (now contains both mechanical and deep findings). This is the final output — make it valuable. - -#### Part 1: What was generated - -List the generated artefacts with counts: -- Blueprint sections populated (out of total) -- Components discovered -- Enforcement rules generated -- Per-folder CLAUDE.md files created -- Rule files in `.claude/rules/` - -#### Part 2: Architecture Summary - -From the blueprint, summarize in 5-10 lines: -- **Architecture style** (from `meta.architecture_style`) -- **Key components** (top 5-7 from `components.components` — name + one-line responsibility). In comprehensive depth (`DEPTH=comprehensive`), list all (no top-N cap). -- **Technology stack highlights** (from `technology.stack` — framework, language, key libs) -- **Key decisions** (from `decisions.key_decisions` — the 2-3 most impactful, one line each). In comprehensive depth (`DEPTH=comprehensive`), list all (no top-N cap). - -#### Part 3: Architecture Health Assessment - -Rate and explain each dimension (use these exact labels: Strong / Adequate / Weak / Not assessed): - -1. **Separation of concerns** — Are layers/modules clearly bounded? Do components have single responsibilities? Any god classes or circular dependencies? -2. **Dependency direction** — Do dependencies flow in one direction? Are domain/core layers independent of infrastructure? Any inverted or tangled dependencies? -3. **Pattern consistency** — Is the same pattern used consistently across similar components? Are there one-off deviations that break the uniformity? -4. **Testability** — Is the architecture conducive to testing? Can components be tested in isolation? Are external dependencies injectable? -5. **Change impact radius** — When a component changes, how many others are affected? Are changes localised or do they ripple? - -Base every rating on actual evidence from the blueprint and drift findings — reference specific components, patterns, or findings. If the blueprint lacks data for a dimension, say "Not assessed" rather than guessing. - -#### Part 4: Architectural Drift - -Present ALL findings — mechanical and deep together, organized by severity (errors first). - -**Deep architectural findings** (from AI analysis): -- For each: the file, which decision/pattern it violates, the evidence, and why it matters -- Group related findings (e.g., multiple files violating the same decision) - -**Mechanical findings** (from script): -- Pattern divergences, dependency violations, naming violations, structural outliers, anti-pattern clusters -- For each: what diverged, why it matters, suggested action - -If 0 findings, say so — that's a positive signal. - -#### Part 5: Top Risks & Recommendations - -Synthesize from pitfalls, trade-offs, drift findings (both mechanical and deep), and your observations. List the **3-5 most important architectural risks**, ordered by impact (in comprehensive depth (`DEPTH=comprehensive`), list all — no top-N cap): -- What the risk is (one sentence) -- Where it manifests (specific components/files/drift findings) -- What to watch for going forward - -#### Part 6: Semantic Duplication - -**This is a critical section.** The mechanical verbosity score (0-1) only catches exact line-for-line clones. AI agents frequently create near-identical functions with slightly different names, signatures, or types — the verbosity metric completely misses these. - -Present the `semantic_duplication` findings from the deep drift analysis. If the drift agent found none, **do your own quick check now**: scan the skeletons for functions with similar names (e.g., `getText`/`getTexts`, `loadUser`/`fetchUser`, `formatDate` in multiple files, `handleError` reimplemented per-module). Read suspicious pairs and confirm whether the logic is duplicated. - -For each confirmed duplicate group: -- The canonical function (the one that should be the shared version) -- The duplicates: which files, what differs (just the signature? types? minor logic?) -- Whether they could be consolidated - -Present in the health table as: -``` -| Semantic duplication | N groups found | See Part 6 for details | -``` - -If genuinely none found after checking, say "No semantic duplication detected after AI analysis." - -**Health scores** from Phase 0 have been saved to `.archie/health_history.json` for trending. Note: the verbosity metric is mechanical (exact line clones only) — the semantic duplication analysis in Part 6 above is the AI-powered complement. - -### Phase 4: Persist findings to `.archie/scan_report.md` - -The Phase 3 synthesis above is valuable but ephemeral — it only exists in the chat output. `{{COMMAND_PREFIX}}archie-share` (and future trending runs of `{{COMMAND_PREFIX}}archie-deep-scan`) need the findings on disk. Write the same content to `.archie/scan_report.md` as ranked findings. - -Check whether a prior scan report exists (for resolved/new/recurring classification): -```bash -test -f "$PROJECT_ROOT/.archie/scan_report.md" && echo "PRIOR_REPORT_EXISTS" || echo "FIRST_BASELINE" -``` - -If `FIRST_BASELINE` (no prior scan_report.md): all findings are tagged **NEW (baseline)**. If `PRIOR_REPORT_EXISTS`: compare against the prior file's Findings section and classify each as **NEW**, **RECURRING**, or **RESOLVED**. - -Read `$PROJECT_ROOT/.archie/health.json` for precise numeric values and `$PROJECT_ROOT/.archie/health_history.json` to compute trends (previous run values vs. current). - -Write `$PROJECT_ROOT/.archie/scan_report.md` using the template at -`{{WORKFLOW_ROOT}}/deep-scan/templates/scan-report.md` (path relative -to the project root). Read that file first if you haven't already, then -substitute the project-specific values into the placeholders before writing -the final report. diff --git a/archie/assets/workflow/deep-scan/steps/step-9-finalize.md b/archie/assets/workflow/deep-scan/steps/step-9-finalize.md new file mode 100644 index 00000000..8037b272 --- /dev/null +++ b/archie/assets/workflow/deep-scan/steps/step-9-finalize.md @@ -0,0 +1,70 @@ +## Step 9: Finalize — health metrics, baseline, telemetry + +**Telemetry:** +```bash +python3 .archie/telemetry.py mark "$PROJECT_ROOT" deep-scan finalize +TELEMETRY_STEP9_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +``` + +**If START_STEP > 9, skip this step.** + +### Phase 1: Health measurement + +```bash +python3 .archie/measure_health.py "$PROJECT_ROOT" > "$PROJECT_ROOT/.archie/health.json" 2>/dev/null +``` + +Save health scores to history for trending: + +```bash +python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep +``` + +### Phase 2: Mark the run complete + +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 +``` + +Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE +``` +(Replace SCAN_MODE with the actual mode — "full" or "incremental") + +### Phase 3: Write telemetry + +Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. + +If the Intent Layer was skipped (INTENT_LAYER=no), mark it so explicitly: + +```bash +if [ "$INTENT_LAYER" = "no" ]; then + python3 .archie/telemetry.py extra "$PROJECT_ROOT" intent_layer skipped=true +fi +``` + +Then flush the in-flight file into the final `.archie/telemetry/deep-scan_.json`: + +```bash +python3 .archie/telemetry.py finish "$PROJECT_ROOT" +python3 .archie/telemetry.py write "$PROJECT_ROOT" +``` + +`write` auto-closes any still-open step with `now`, emits the final timestamped JSON, then deletes `_current_run.json` so the next deep-scan starts fresh. If telemetry fails for any reason, do not abort — telemetry is informational only. + +**Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. + +### Phase 4: Closing summary + +Present a short wrap-up to the user — a receipt, not a report (10 lines or fewer). State what the scan produced, with counts read via the allowlisted inspect commands (NEVER inline Python): + +```bash +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" blueprint.json --query '.components.components|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" rules.json --query '.rules|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" findings.json --query '.findings|length' +``` + +Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (you know this count from Step 7), and findings tracked in `.archie/findings.json`. Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. + +End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** diff --git a/archie/assets/workflow/deep-scan/templates/scan-report.md b/archie/assets/workflow/deep-scan/templates/scan-report.md deleted file mode 100644 index ec96d672..00000000 --- a/archie/assets/workflow/deep-scan/templates/scan-report.md +++ /dev/null @@ -1,74 +0,0 @@ -# Archie Scan Report -> Deep scan baseline | | functions / LOC analyzed | baseline run - -## Architecture Overview - -<2-3 paragraphs from Part 2: architecture style, key components, most important decisions. Prose, not bullets.> - -## Health Scores - -| Metric | Current | Previous | Trend | What it means | -|--------|--------:|---------:|------:|---------------| -| Erosion | | | | | -| Gini | | | | | -| Top-20% | | | | | -| Verbosity | | | | | -| LOC | | | | | - - - -### Complexity Trajectory - - -## Findings - -Ranked by severity, grouped by novelty. - -### NEW (first observed this scan) - - -### RECURRING (previously documented, still present) - - -### RESOLVED - - -### Data Architecture - - -## Proposed Rules - - -``` - -Sources for Findings: -- `drift_report.json` — mechanical and deep drift findings from Phase 1 and 2 -- `blueprint.json` — `pitfalls` (each causal chain becomes a finding), `decisions.trade_offs` with violated `violation_signals` (if any appear in drift_report) -- Top complexity offenders from `health.json` (only if CC ≥ 15 or a cluster — don't list every high-CC function as a finding) - -Severity mapping: -- `error` — decision violations, inverted dependencies, cycles across architectural boundaries -- `warn` — pattern erosion, god-objects, pitfalls currently manifesting, trade-offs actively undermined -- `info` — structural observations (dependency magnets, high fan-in nodes) that aren't currently broken - -Confidence: carry forward from drift findings when available; otherwise use 0.8-0.95 for findings grounded in direct code reading, lower for inferred ones. - -Verify the write: -```bash -test -s "$PROJECT_ROOT/.archie/scan_report.md" && wc -l "$PROJECT_ROOT/.archie/scan_report.md" -``` - -Expected: non-empty file with at least 30 lines. - -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 -``` - -Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE -``` -(Replace SCAN_MODE with the actual mode — "full" or "incremental") - -End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** - diff --git a/archie/install.py b/archie/install.py index 63c3296e..7c19ec86 100644 --- a/archie/install.py +++ b/archie/install.py @@ -43,7 +43,7 @@ def _resolve_targets(requested: list[str] | None, connectors: list[Connector]) - # Analysis pipeline (referenced by SKILL bodies via `python3 .archie/`) "scanner.py", "renderer.py", "validate.py", "intent_layer.py", "finalize.py", "merge.py", "measure_health.py", "detect_cycles.py", - "drift.py", "extract_output.py", "arch_review.py", "align_check.py", + "extract_output.py", "arch_review.py", "align_check.py", "check_rules.py", "code_shape.py", "rule_index.py", "lint_gate.py", "agent_cli.py", "verify_findings.py", "apply_verdicts.py", "migrate_blueprint_rules.py", "rule_kinds.py", "backfill_kinds.py", @@ -87,6 +87,16 @@ def _clean_legacy_layout(project_root: Path) -> None: except OSError: pass + # Retired pipeline scripts — remove on upgrade so stale copies don't linger + # in .archie/. (The npm installer wipes all .py files before copying; this + # pip path copies over without sweeping, so retired names need an explicit + # delete.) + for retired in ("drift.py",): + try: + (project_root / ".archie" / retired).unlink() + except OSError: + pass + current_command_names = {c.name for c in COMMANDS} # Sweep stale Claude command shims (.claude/commands/archie-X.md). diff --git a/archie/manifest_data.py b/archie/manifest_data.py index 86788306..c070fcc5 100644 --- a/archie/manifest_data.py +++ b/archie/manifest_data.py @@ -126,17 +126,18 @@ claude_glob="Bash(python3 -c *)", justification="Inline Python — Archie workflow uses for JSON inspection", ), - # Codex spawning Codex for the headless finding-verifier in Step 9 drift. - # verify_findings.py shells out to `codex exec --sandbox read-only …` - # (see archie/standalone/agent_cli.py::_run_codex). Without this rule the - # parent Codex session would prompt mid-Step-9 on every verified finding. + # Codex spawning Codex for the headless finding-verifier (deep-scan Step 5, + # after finalize writes findings.json). verify_findings.py shells out to + # `codex exec --sandbox read-only …` (see agent_cli.py::_run_codex). + # Without this rule the parent Codex session would prompt mid-Step-5 on + # every verified finding. CommandRule( name="codex-exec", codex_pattern=("codex", "exec"), claude_glob="Bash(codex exec *)", justification=( "verify_findings.py spawns `codex exec` for per-finding model " - "calls during Step 9 drift verification" + "calls during Step 5 finding verification" ), ), # Filesystem prep / inspection diff --git a/archie/standalone/_common.py b/archie/standalone/_common.py index 5c673c1b..6a7c0c3c 100644 --- a/archie/standalone/_common.py +++ b/archie/standalone/_common.py @@ -240,7 +240,7 @@ def is_ignored(self, rel_path: str) -> bool: Gitignore semantics: a path is also ignored when any ANCESTOR directory matches a dir pattern (``vendor/`` ignores ``vendor/b.py``). os.walk callers get this for free via directory pruning; full-path callers - (e.g. drift's git-log list, where the file may no longer exist on disk) + (e.g. a git-log file list, where the file may no longer exist on disk) rely on this ancestor walk. """ rel_path = rel_path.replace(os.sep, "/") diff --git a/archie/standalone/drift.py b/archie/standalone/drift.py deleted file mode 100644 index 2d6bc9d0..00000000 --- a/archie/standalone/drift.py +++ /dev/null @@ -1,722 +0,0 @@ -#!/usr/bin/env python3 -"""Archie drift detector — finds architectural divergences and outliers. - -Compares per-folder enrichments and file structure against the blueprint's -dominant patterns to surface: pattern inconsistencies, naming violations, -dependency direction breaches, and structural outliers. - -Run: - python3 drift.py /path/to/repo - -Output: JSON drift report to stdout, human summary to stderr. - -Zero dependencies beyond Python 3.9+ stdlib. -""" -from __future__ import annotations - -import json -import re -import sys -from collections import Counter, defaultdict -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -from _common import _load_json # noqa: E402 - - -# --------------------------------------------------------------------------- -# 1. Pattern consistency — which folders diverge from dominant patterns -# --------------------------------------------------------------------------- - -def check_pattern_consistency(root: Path) -> list[dict]: - """Find folders whose patterns diverge from the codebase-wide norms.""" - enrichments_dir = root / ".archie" / "enrichments" - if not enrichments_dir.is_dir(): - return [] - - # Collect all patterns across all folders - folder_patterns: dict[str, list[str]] = {} - all_enrichments: dict[str, dict] = {} - for json_file in sorted(enrichments_dir.iterdir()): - if not json_file.name.endswith(".json"): - continue - try: - data = json.loads(json_file.read_text()) - if isinstance(data, dict): - all_enrichments.update(data) - except (json.JSONDecodeError, OSError): - continue - - if not all_enrichments: - return [] - - # Extract pattern names per folder - pattern_counter: Counter = Counter() - for folder, info in all_enrichments.items(): - if not isinstance(info, dict): - continue - patterns = info.get("patterns", []) - names = [] - for p in patterns: - if isinstance(p, dict) and p.get("name"): - names.append(p["name"].lower().strip()) - elif isinstance(p, str): - names.append(p.lower().strip()) - folder_patterns[folder] = names - for n in names: - pattern_counter[n] += 1 - - if not pattern_counter: - return [] - - # Find dominant patterns (used in >25% of folders) - threshold = max(2, len(folder_patterns) * 0.25) - dominant = {name for name, count in pattern_counter.items() if count >= threshold} - - # Group folders by their parent to find sibling divergences - siblings: dict[str, list[str]] = defaultdict(list) - for folder in folder_patterns: - parent = str(Path(folder).parent) - siblings[parent].append(folder) - - findings = [] - - # Check sibling consistency — folders under the same parent should use - # similar patterns - for parent, sibs in siblings.items(): - if len(sibs) < 2: - continue - # Collect patterns used by siblings - sib_patterns: Counter = Counter() - for s in sibs: - for p in folder_patterns.get(s, []): - sib_patterns[p] += 1 - # Sibling-dominant: used by >50% of siblings - sib_threshold = len(sibs) * 0.5 - sib_dominant = {name for name, count in sib_patterns.items() if count >= sib_threshold} - - for s in sibs: - s_pats = set(folder_patterns.get(s, [])) - missing = sib_dominant - s_pats - if missing and s_pats: # has some patterns but missing dominant ones - findings.append({ - "type": "pattern_divergence", - "folder": s, - "message": f"Siblings under {parent}/ mostly use [{', '.join(sorted(missing))}] but this folder does not", - "expected": sorted(sib_dominant), - "actual": sorted(s_pats), - "severity": "warn", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 2. Naming convention violations -# --------------------------------------------------------------------------- - -def check_naming_conventions(root: Path) -> list[dict]: - """Check files against naming conventions from the blueprint.""" - bp = _load_json(root / ".archie" / "blueprint.json") - rules = bp.get("architecture_rules", {}) - conventions = rules.get("naming_conventions", []) - scan = _load_json(root / ".archie" / "scan.json") - files = [f.get("path", "") for f in scan.get("file_tree", [])] - - if not conventions or not files: - return [] - - findings = [] - for conv in conventions: - if not isinstance(conv, dict): - continue - scope = conv.get("scope", "") - pattern = conv.get("pattern", "") - description = conv.get("description", "") - examples = conv.get("examples", []) - - if not pattern or not scope: - continue - - # Try to build a regex from the pattern description - # Common patterns: PascalCase, camelCase, snake_case, kebab-case - regex = None - pat_lower = pattern.lower() - if "pascalcase" in pat_lower or "pascal case" in pat_lower: - regex = r'^[A-Z][a-zA-Z0-9]*' - elif "camelcase" in pat_lower or "camel case" in pat_lower: - regex = r'^[a-z][a-zA-Z0-9]*' - elif "snake_case" in pat_lower or "snake case" in pat_lower: - regex = r'^[a-z][a-z0-9_]*' - elif "kebab" in pat_lower: - regex = r'^[a-z][a-z0-9\-]*' - - if not regex: - continue - - # Find files that match the scope but violate the pattern - violations = [] - scope_lower = scope.lower() - for fp in files: - # Check if file is in scope — match path segments, not substrings - fp_lower = fp.lower() - if not (fp_lower.startswith(scope_lower + "/") or - ("/" + scope_lower + "/") in fp_lower or - fp_lower == scope_lower): - continue - filename = fp.rsplit("/", 1)[-1] if "/" in fp else fp - name_part = filename.rsplit(".", 1)[0] if "." in filename else filename - if not re.match(regex, name_part): - violations.append(fp) - - if violations and len(violations) <= len(files) * 0.3: - # Only report if it's a minority (actual outliers, not a bad rule) - findings.append({ - "type": "naming_violation", - "convention": description or pattern, - "scope": scope, - "violating_files": violations if _COMPREHENSIVE else violations[:10], - "count": len(violations), - "severity": "info", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 3. Dependency direction violations -# --------------------------------------------------------------------------- - -def check_dependency_direction(root: Path) -> list[dict]: - """Check import graph against component dependency declarations.""" - bp = _load_json(root / ".archie" / "blueprint.json") - scan = _load_json(root / ".archie" / "scan.json") - import_graph = scan.get("import_graph", {}) - - comps_raw = bp.get("components", {}) - components = comps_raw.get("components", []) if isinstance(comps_raw, dict) else comps_raw if isinstance(comps_raw, list) else [] - - if not import_graph or not components: - return [] - - # Build component location map and allowed dependency map - comp_by_loc: dict[str, dict] = {} - for comp in components: - loc = (comp.get("location") or "").rstrip("/") - if loc: - comp_by_loc[loc] = comp - - findings = [] - - # For each component, check if files import from components NOT in depends_on - for comp in components: - loc = (comp.get("location") or "").rstrip("/") - if not loc: - continue - allowed_deps = set(comp.get("depends_on", [])) - comp_name = comp.get("name", loc) - - # Find all imports from files in this component - for file_path, imports in import_graph.items(): - if not (file_path.startswith(loc + "/") or file_path == loc): - continue - for imp in imports: - imp_path = imp.replace(".", "/") - # Check if this import lands in another component - for other_loc, other_comp in comp_by_loc.items(): - if other_loc == loc: - continue - other_name = other_comp.get("name", other_loc) - if imp_path.startswith(other_loc) or other_loc in imp_path: - if other_name not in allowed_deps and other_loc not in allowed_deps: - findings.append({ - "type": "dependency_violation", - "from_component": comp_name, - "to_component": other_name, - "file": file_path, - "import": imp, - "message": f"{comp_name} imports from {other_name} but does not declare it as a dependency", - "severity": "warn", - }) - - # Deduplicate by (from, to) pair — keep first occurrence - seen = set() - deduped = [] - for f in findings: - key = (f["from_component"], f["to_component"]) - if key not in seen: - seen.add(key) - deduped.append(f) - - return deduped - - -# --------------------------------------------------------------------------- -# 4. Structural outliers — folders organized differently from siblings -# --------------------------------------------------------------------------- - -def check_structural_outliers(root: Path) -> list[dict]: - """Find folders that are structurally different from their siblings.""" - scan = _load_json(root / ".archie" / "scan.json") - files = scan.get("file_tree", []) - - if not files: - return [] - - # Count files per directory and collect extensions - dir_file_counts: dict[str, int] = defaultdict(int) - dir_extensions: dict[str, Counter] = defaultdict(Counter) - for f in files: - p = f.get("path", "") - if "/" not in p: - continue - parent = str(Path(p).parent) - dir_file_counts[parent] += 1 - ext = f.get("extension", "") - if ext: - dir_extensions[parent][ext] += 1 - - # Group by grandparent to find sibling directories - siblings: dict[str, list[str]] = defaultdict(list) - for d in dir_file_counts: - grandparent = str(Path(d).parent) - siblings[grandparent].append(d) - - findings = [] - for grandparent, sibs in siblings.items(): - if len(sibs) < 3: - continue - - # Check for file count outliers (>3 standard deviations) - counts = [dir_file_counts[s] for s in sibs] - if not counts: - continue - avg = sum(counts) / len(counts) - if avg == 0: - continue - variance = sum((c - avg) ** 2 for c in counts) / len(counts) - std = variance ** 0.5 - - if std == 0: - continue - - for s in sibs: - count = dir_file_counts[s] - if std > 0 and abs(count - avg) > 3 * std and count > avg: - findings.append({ - "type": "structural_outlier", - "folder": s, - "message": f"Has {count} files — significantly more than sibling average ({avg:.0f}). May be a god-folder that should be split.", - "severity": "info", - }) - - # Check for extension mismatches — siblings should have similar file types - sib_exts: Counter = Counter() - for s in sibs: - for ext in dir_extensions[s]: - sib_exts[ext] += 1 - # Dominant extension: used by >60% of siblings - dominant_exts = {ext for ext, cnt in sib_exts.items() if cnt >= len(sibs) * 0.6} - - for s in sibs: - s_exts = set(dir_extensions[s].keys()) - unexpected = s_exts - dominant_exts - if unexpected and dominant_exts and len(unexpected) > len(dominant_exts): - findings.append({ - "type": "structural_outlier", - "folder": s, - "message": f"Uses [{', '.join(sorted(unexpected))}] while siblings mostly use [{', '.join(sorted(dominant_exts))}]", - "severity": "info", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 5. Anti-pattern clusters — folders with many anti-patterns -# --------------------------------------------------------------------------- - -def check_antipattern_clusters(root: Path) -> list[dict]: - """Find folders with an unusual density of anti-patterns.""" - enrichments_dir = root / ".archie" / "enrichments" - if not enrichments_dir.is_dir(): - return [] - - all_enrichments: dict[str, dict] = {} - for json_file in sorted(enrichments_dir.iterdir()): - if not json_file.name.endswith(".json"): - continue - try: - data = json.loads(json_file.read_text()) - if isinstance(data, dict): - all_enrichments.update(data) - except (json.JSONDecodeError, OSError): - continue - - if not all_enrichments: - return [] - - # Count anti-patterns per folder - ap_counts = {} - for folder, info in all_enrichments.items(): - if not isinstance(info, dict): - continue - anti = info.get("anti_patterns", []) - if anti: - ap_counts[folder] = len(anti) - - if not ap_counts: - return [] - - avg = sum(ap_counts.values()) / len(ap_counts) if ap_counts else 0 - threshold = max(avg * 2, 4) - - findings = [] - for folder, count in sorted(ap_counts.items(), key=lambda x: -x[1]): - if count >= threshold: - anti = all_enrichments[folder].get("anti_patterns", []) - findings.append({ - "type": "antipattern_cluster", - "folder": folder, - "count": count, - "anti_patterns": anti if _COMPREHENSIVE else anti[:5], - "message": f"Has {count} anti-patterns (avg: {avg:.1f}) — high-risk area for accidental violations", - "severity": "warn", - }) - - return findings - - -# --------------------------------------------------------------------------- -# Main report -# --------------------------------------------------------------------------- - -_SECTIONS = [ - ("pattern_divergences", "Pattern Divergences"), - ("dependency_violations", "Dependency Violations"), - ("naming_violations", "Naming Violations"), - ("structural_outliers", "Structural Outliers"), - ("antipattern_clusters", "Anti-Pattern Clusters"), -] - - -def generate_drift_report(root: Path) -> dict: - """Generate a full drift report.""" - from datetime import datetime, timezone - - report = { - "pattern_divergences": check_pattern_consistency(root), - "naming_violations": check_naming_conventions(root), - "dependency_violations": check_dependency_direction(root), - "structural_outliers": check_structural_outliers(root), - "antipattern_clusters": check_antipattern_clusters(root), - } - - # Summary counts - total = sum(len(v) for v in report.values()) - warns = sum(1 for v in report.values() for f in v if f.get("severity") == "warn") - infos = total - warns - - report["summary"] = { - "total_findings": total, - "warnings": warns, - "informational": infos, - "checks_run": [k for k, v in report.items() if k != "summary"], - "timestamp": datetime.now(timezone.utc).isoformat(), - } - - return report - - -def print_human_summary(report: dict): - """Print human-readable drift summary to stderr.""" - summary = report.get("summary", {}) - total = summary.get("total_findings", 0) - - print(f"\nDrift Analysis: {total} findings", file=sys.stderr) - - for section, label in _SECTIONS: - items = report.get(section, []) - if not items: - continue - print(f"\n {label} ({len(items)}):", file=sys.stderr) - for item in items[:5]: - sev = item.get("severity", "info").upper() - msg = item.get("message", "") - folder = item.get("folder", "") - if folder: - print(f" [{sev}] {folder}: {msg}", file=sys.stderr) - else: - print(f" [{sev}] {msg}", file=sys.stderr) - if len(items) > 5: - print(f" ... and {len(items) - 5} more", file=sys.stderr) - - if total == 0: - print(" No architectural drift detected.", file=sys.stderr) - print("", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# Snapshot history -# --------------------------------------------------------------------------- - -_HISTORY_DIR = "drift_history" - - -def _save_snapshot(root: Path, report: dict): - """Save a timestamped drift snapshot for future diffing.""" - from datetime import datetime, timezone - - history_dir = root / ".archie" / _HISTORY_DIR - history_dir.mkdir(parents=True, exist_ok=True) - - ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - snapshot_path = history_dir / f"drift_{ts}.json" - snapshot_path.write_text(json.dumps(report, indent=2)) - - # Also update the "latest" symlink / copy for easy access - latest_path = history_dir / "latest.json" - latest_path.write_text(json.dumps(report, indent=2)) - - return snapshot_path - - -def _load_previous_snapshot(root: Path) -> dict | None: - """Load the most recent snapshot before the current one.""" - history_dir = root / ".archie" / _HISTORY_DIR - if not history_dir.is_dir(): - return None - - snapshots = sorted( - [f for f in history_dir.iterdir() if f.name.startswith("drift_") and f.name.endswith(".json")], - key=lambda f: f.name, - ) - - # Return second-to-last (the previous run), since the current run - # may have already been saved - if len(snapshots) >= 2: - try: - return json.loads(snapshots[-2].read_text()) - except (json.JSONDecodeError, OSError): - pass - return None - - -# --------------------------------------------------------------------------- -# Diff engine -# --------------------------------------------------------------------------- - -def _finding_key(finding: dict) -> str: - """Create a stable identity key for a finding so we can diff across runs.""" - ftype = finding.get("type", "") - folder = finding.get("folder", "") - # For dependency violations, include both ends - from_comp = finding.get("from_component", "") - to_comp = finding.get("to_component", "") - # For naming violations, include scope - scope = finding.get("scope", "") - convention = finding.get("convention", "") - # Include message prefix to disambiguate same-folder findings - # (e.g., two structural_outlier findings for the same folder: file count vs extension) - msg = finding.get("message", "")[:80] - - parts = [ftype, folder, from_comp, to_comp, scope, convention, msg] - return "|".join(p for p in parts if p) - - -def compute_diff(previous: dict, current: dict) -> dict: - """Compare two drift reports and return new, resolved, and persisting findings.""" - diff_result = { - "new": [], # in current but not in previous - "resolved": [], # in previous but not in current - "persisting": [], # in both - } - - prev_ts = previous.get("summary", {}).get("timestamp", "unknown") - curr_ts = current.get("summary", {}).get("timestamp", "unknown") - - # Collect all findings from both reports, keyed - prev_findings: dict[str, dict] = {} - curr_findings: dict[str, dict] = {} - - for section, _ in _SECTIONS: - for f in previous.get(section, []): - prev_findings[_finding_key(f)] = f - for f in current.get(section, []): - curr_findings[_finding_key(f)] = f - - prev_keys = set(prev_findings.keys()) - curr_keys = set(curr_findings.keys()) - - for key in sorted(curr_keys - prev_keys): - diff_result["new"].append(curr_findings[key]) - for key in sorted(prev_keys - curr_keys): - diff_result["resolved"].append(prev_findings[key]) - for key in sorted(curr_keys & prev_keys): - diff_result["persisting"].append(curr_findings[key]) - - diff_result["summary"] = { - "previous_timestamp": prev_ts, - "current_timestamp": curr_ts, - "previous_total": previous.get("summary", {}).get("total_findings", 0), - "current_total": current.get("summary", {}).get("total_findings", 0), - "new_findings": len(diff_result["new"]), - "resolved_findings": len(diff_result["resolved"]), - "persisting_findings": len(diff_result["persisting"]), - } - - return diff_result - - -def print_diff_summary(diff_result: dict): - """Print human-readable diff to stderr.""" - s = diff_result["summary"] - prev_total = s["previous_total"] - curr_total = s["current_total"] - delta = curr_total - prev_total - - delta_str = f"+{delta}" if delta > 0 else str(delta) - print(f"\nDrift Diff: {prev_total} -> {curr_total} ({delta_str})", file=sys.stderr) - print(f" Previous: {s['previous_timestamp']}", file=sys.stderr) - print(f" Current: {s['current_timestamp']}", file=sys.stderr) - - new = diff_result["new"] - resolved = diff_result["resolved"] - persisting = diff_result["persisting"] - - if new: - print(f"\n NEW ({len(new)}):", file=sys.stderr) - for item in new: - sev = item.get("severity", "info").upper() - folder = item.get("folder", "") - msg = item.get("message", "") - print(f" + [{sev}] {folder}: {msg}", file=sys.stderr) - - if resolved: - print(f"\n RESOLVED ({len(resolved)}):", file=sys.stderr) - for item in resolved: - sev = item.get("severity", "info").upper() - folder = item.get("folder", "") - msg = item.get("message", "") - print(f" - [{sev}] {folder}: {msg}", file=sys.stderr) - - if persisting: - print(f"\n PERSISTING ({len(persisting)})", file=sys.stderr) - - if not new and not resolved: - print("\n No changes since last run.", file=sys.stderr) - - print("", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def cmd_run(root: Path): - """Run drift detection, save snapshot, auto-diff against previous, print summary.""" - report = generate_drift_report(root) - print_human_summary(report) - - # Save report and snapshot - out_path = root / ".archie" / "drift_report.json" - out_path.write_text(json.dumps(report, indent=2)) - snapshot_path = _save_snapshot(root, report) - print(f"Saved: {out_path}", file=sys.stderr) - print(f"Snapshot: {snapshot_path}", file=sys.stderr) - - # Auto-diff against previous snapshot if one exists - previous = _load_previous_snapshot(root) - if previous: - diff_result = compute_diff(previous, report) - print_diff_summary(diff_result) - - diff_path = root / ".archie" / "drift_diff.json" - diff_path.write_text(json.dumps(diff_result, indent=2)) - - # Output JSON to stdout - print(json.dumps(report, indent=2)) - - -def cmd_history(root: Path): - """List all drift snapshots with finding counts.""" - history_dir = root / ".archie" / _HISTORY_DIR - if not history_dir.is_dir(): - print("No drift history found.", file=sys.stderr) - return - - snapshots = sorted( - [f for f in history_dir.iterdir() if f.name.startswith("drift_") and f.name.endswith(".json")], - key=lambda f: f.name, - ) - - if not snapshots: - print("No drift snapshots found.", file=sys.stderr) - return - - entries = [] - prev_total = None - for snap in snapshots: - try: - data = json.loads(snap.read_text()) - s = data.get("summary", {}) - total = s.get("total_findings", 0) - warns = s.get("warnings", 0) - ts = s.get("timestamp", "") - - delta = "" - if prev_total is not None: - d = total - prev_total - delta = f" (+{d})" if d > 0 else f" ({d})" if d < 0 else " (=)" - prev_total = total - - entries.append({ - "file": snap.name, - "timestamp": ts, - "total": total, - "warnings": warns, - "delta": delta, - }) - except (json.JSONDecodeError, OSError): - continue - - print(f"\nDrift History ({len(entries)} snapshots):\n", file=sys.stderr) - for e in entries: - print(f" {e['file']} {e['total']:3d} findings ({e['warnings']} warn){e['delta']}", file=sys.stderr) - print("", file=sys.stderr) - - # JSON to stdout - print(json.dumps(entries, indent=2)) - - -# Comprehensive-depth switch: lift the report detail-list caps (full lists, no -# [:10]/[:5]). Read from run state; *count fields are unaffected. -_COMPREHENSIVE = False - - -def _is_comprehensive(root) -> bool: - try: - st = _load_json(Path(root) / ".archie" / "deep_scan_state.json") - return (st.get("run_context") or {}).get("depth") == "comprehensive" - except Exception: - return False - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage:", file=sys.stderr) - print(" python3 drift.py /path/to/repo — run drift detection (auto-diffs if previous exists)", file=sys.stderr) - print(" python3 drift.py history /path/to/repo — list all snapshots", file=sys.stderr) - sys.exit(1) - - # Parse subcommand - if sys.argv[1] == "history": - if len(sys.argv) < 3: - print("Usage: python3 drift.py history /path/to/repo", file=sys.stderr) - sys.exit(1) - root = Path(sys.argv[2]).resolve() - cmd_history(root) - else: - root = Path(sys.argv[1]).resolve() - _COMPREHENSIVE = _is_comprehensive(root) or ("--comprehensive" in sys.argv) - cmd_run(root) diff --git a/archie/standalone/extract_output.py b/archie/standalone/extract_output.py index d0db0906..89129e7e 100644 --- a/archie/standalone/extract_output.py +++ b/archie/standalone/extract_output.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 """Archie extract_output — robust extraction of agent output for the pipeline. -Replaces all inline python3 -c one-liners in archie-init.md and archie-drift.md. +Replaces inline python3 -c one-liners in the workflow files. Uses merge.extract_json_from_text to handle conversation envelopes, code fences, and AI escape issues. Subcommands: rules — extract rules JSON from agent output - deep-drift — extract deep findings, merge into drift report - recent-files — print source file paths from scan.json save-duplications — write .archie/semantic_duplications.json Zero dependencies beyond Python 3.9+ stdlib + sibling merge.py. @@ -127,68 +125,6 @@ def cmd_rules(input_file: str, output_path: str): print(f"Saved {len(new_rules)} rules to {output_path}", file=sys.stderr) -# --------------------------------------------------------------------------- -# deep-drift — extract deep findings and merge into drift report -# --------------------------------------------------------------------------- - -def cmd_deep_drift(input_file: str, report_path: str): - """Extract deep architectural findings from agent output, merge into drift report.""" - text = Path(input_file).read_text() - data = extract_json_from_text(text) - - if not data: - print("Warning: could not extract deep findings", file=sys.stderr) - sys.exit(1) - - report_file = Path(report_path) - if report_file.exists(): - report = json.loads(report_file.read_text()) - else: - report = {"summary": {"total_findings": 0, "warnings": 0}} - - deep_findings = data.get("deep_findings", []) - report["deep_findings"] = deep_findings - - s = report["summary"] - deep_count = len(deep_findings) - s["deep_findings"] = deep_count - s["total_findings"] = s.get("total_findings", 0) + deep_count - s["warnings"] = s.get("warnings", 0) + sum( - 1 for f in deep_findings if f.get("severity") == "warn" - ) - - report_file.write_text(json.dumps(report, indent=2)) - print(f"Added {deep_count} deep findings to {report_path}", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# recent-files — print source file paths from scan.json -# --------------------------------------------------------------------------- - -_SOURCE_EXTENSIONS = { - ".kt", ".java", ".swift", ".ts", ".tsx", ".js", ".jsx", - ".py", ".go", ".rs", ".rb", ".dart", ".cs", ".cpp", ".c", ".h", - ".m", ".scala", ".clj", ".ex", ".exs", ".zig", ".lua", -} - - -def cmd_recent_files(scan_json: str): - """Print source file paths from scan.json, one per line.""" - data = json.loads(Path(scan_json).read_text()) - file_tree = data.get("file_tree", []) - - count = 0 - for f in file_tree: - ext = f.get("extension", "") - if ext in _SOURCE_EXTENSIONS: - print(f["path"]) - count += 1 - if count >= 100: - break - - print(f"Listed {count} source files", file=sys.stderr) - - # --------------------------------------------------------------------------- # save-duplications — extract Agent C's duplications and write to .archie/ # --------------------------------------------------------------------------- @@ -230,8 +166,6 @@ def cmd_save_duplications(agent_c_file: str, project_root: str): if len(sys.argv) < 2: print("Usage:", file=sys.stderr) print(" python3 extract_output.py rules ", file=sys.stderr) - print(" python3 extract_output.py deep-drift ", file=sys.stderr) - print(" python3 extract_output.py recent-files ", file=sys.stderr) print(" python3 extract_output.py save-duplications ", file=sys.stderr) sys.exit(1) @@ -243,18 +177,6 @@ def cmd_save_duplications(agent_c_file: str, project_root: str): sys.exit(1) cmd_rules(sys.argv[2], sys.argv[3]) - elif subcmd == "deep-drift": - if len(sys.argv) < 4: - print("Usage: extract_output.py deep-drift ", file=sys.stderr) - sys.exit(1) - cmd_deep_drift(sys.argv[2], sys.argv[3]) - - elif subcmd == "recent-files": - if len(sys.argv) < 3: - print("Usage: extract_output.py recent-files ", file=sys.stderr) - sys.exit(1) - cmd_recent_files(sys.argv[2]) - elif subcmd == "save-duplications": if len(sys.argv) < 4: print("Usage: extract_output.py save-duplications ", file=sys.stderr) diff --git a/archie/standalone/intent_layer.py b/archie/standalone/intent_layer.py index 2e6e774f..b45c84e6 100644 --- a/archie/standalone/intent_layer.py +++ b/archie/standalone/intent_layer.py @@ -595,7 +595,7 @@ def _load_state() -> dict: _STEP_NAMES = { 1: "scan", 2: "read", 3: "wave1", 4: "merge", 5: "wave2_synthesis", 6: "rule_synthesis", - 7: "intent_layer", 8: "cleanup", 9: "drift", + 7: "intent_layer", 8: "cleanup", 9: "finalize", } step_name = _STEP_NAMES.get(step) if step_name: @@ -2304,26 +2304,6 @@ def cmd_inspect(root: Path, filename: str, query: str | None = None, as_list: bo print(json.dumps(data, indent=2)) -def cmd_filter_ignored(root: Path): - """Read newline-separated repo-relative paths on stdin; print those NOT - ignored by .gitignore/.archieignore. Used by step-9 drift so its git-log - file list honors the ignore system (git does not know .archieignore). - - ``IgnoreMatcher.is_ignored`` applies full gitignore semantics, including - ancestor-directory matches (``vendor/`` hides ``vendor/b.py``).""" - matcher = IgnoreMatcher(root) - - def _is_path_ignored(rel: str) -> bool: - return matcher.is_ignored(rel.replace(os.sep, "/")) - - for line in sys.stdin: - rel = line.strip() - if not rel: - continue - if not _is_path_ignored(rel): - print(rel) - - # --------------------------------------------------------------------------- # CLI entry point # --------------------------------------------------------------------------- @@ -2456,8 +2436,6 @@ def _is_path_ignored(rel: str) -> bool: cmd_inject_scoped(root) elif subcmd == "extract-guardrails": cmd_extract_guardrails(root) - elif subcmd == "filter-ignored": - cmd_filter_ignored(Path(sys.argv[2]).resolve()) elif subcmd == "inspect": if len(sys.argv) < 4: print("Usage: inspect /path/to/repo [--query .key.path] [--list]", file=sys.stderr) diff --git a/archie/standalone/sync.py b/archie/standalone/sync.py index 8b1052f1..1de49f7c 100644 --- a/archie/standalone/sync.py +++ b/archie/standalone/sync.py @@ -333,7 +333,7 @@ def cmd_record(root: Path, input_file: str | None, agent: str, since: str | None "claims": out_claims, } - # 5. Write the versioned change file + latest.json (mirror drift._save_snapshot). + # 5. Write the versioned change file + latest.json (timestamped snapshot history). changes_dir = _changes_dir(root) changes_dir.mkdir(parents=True, exist_ok=True) record["version"] = _next_version(changes_dir) diff --git a/archie/standalone/telemetry.py b/archie/standalone/telemetry.py index 9f99959b..fe0f8307 100644 --- a/archie/standalone/telemetry.py +++ b/archie/standalone/telemetry.py @@ -293,7 +293,7 @@ def _detect_cli() -> str: "rule_synthesis": "Rule synthesis", "intent_layer": "Intent Layer", "cleanup": "Cleanup", - "drift": "Drift & assessment", + "finalize": "Finalize", } # Parallel sub-agent friendly labels (Wave 1 fact agents + Wave 2 reasoning agents). _AGENT_LABELS = { diff --git a/archie/standalone/upload.py b/archie/standalone/upload.py index d02261ef..1a052efa 100644 --- a/archie/standalone/upload.py +++ b/archie/standalone/upload.py @@ -3,7 +3,7 @@ Run: python3 upload.py /path/to/project Reads from .archie/: blueprint.json (required), health.json, scan.json, rules.json, - proposed_rules.json, scan_report.md (all optional). + proposed_rules.json, findings.json (all optional). Prints: shareable URL on success, warning on failure. Zero dependencies beyond Python 3.9+ stdlib. @@ -174,15 +174,11 @@ def build_bundle(project_root: Path) -> dict: if proposed: bundle["rules_proposed"] = proposed - scan_report = _read_text(archie_dir / "scan_report.md") - if scan_report: - bundle["scan_report"] = scan_report - # Structured findings from the shared accumulating store. Gives the share # viewer the 4-field shape (problem_statement/evidence/root_cause/ - # fix_direction) — far richer than the title/description regex-scraped - # from scan_report.md. Old bundles without this still fall back to the - # markdown-parsed findings. + # fix_direction). The legacy scan_report.md bundle field was retired with + # the deep-scan drift step — the viewer treats it as optional and old + # bundles that still carry it keep rendering. findings_store = _read_json(archie_dir / "findings.json") if isinstance(findings_store, dict) and isinstance(findings_store.get("findings"), list): bundle["findings"] = findings_store["findings"] @@ -195,56 +191,22 @@ def build_bundle(project_root: Path) -> dict: if isinstance(c4, dict): bundle["c4"] = c4 - # Structured semantic duplications. Two upstream sources, merged here so - # the share viewer always sees a single authoritative count regardless of - # which scan path produced them: - # - # 1. .archie/semantic_duplications.json — Agent C's output. Present - # when scan_report.md exists (written by deep-scan Phase 4). - # 2. .archie/drift_report.json deep_findings tagged - # "semantic_duplication" — /archie-deep-scan's deep-drift agent. - # This was getting silently dropped: the file existed, the findings - # were tagged, but upload.py never looked at them. Visible symptom - # was "0 semantic reimplementations" on the share cover for projects - # whose dups came from deep-drift only. + # Structured semantic duplications — legacy source only. + # .archie/semantic_duplications.json was Agent C's output from the retired + # scan flow; the deep-scan drift agent (the other historical source, via + # drift_report.json) was removed along with the drift step, and no current + # pipeline emits semantic duplications. Projects that still carry the + # legacy file keep their count on the share cover; everyone else falls back + # to health.json's mechanical line-clone metric. # # Distinct from health.json's textual duplicates (line-identical copy- - # paste). Both sources describe near-twin functions / reimplementations. - duplications: list = [] - saw_any_source = False + # paste): this field describes near-twin functions / reimplementations. sem = _read_json(archie_dir / "semantic_duplications.json") if sem and isinstance(sem.get("duplications"), list): - saw_any_source = True - duplications.extend(sem["duplications"]) - - drift = _read_json(archie_dir / "drift_report.json") - if isinstance(drift, dict): - deep_findings = drift.get("deep_findings") or [] - if isinstance(deep_findings, list): - saw_any_source = True - for f in deep_findings: - if not isinstance(f, dict): - continue - # The deep-drift agent tags entries via a `type` field - # ("semantic_duplication", "missing_pattern", etc.). Some - # older / hand-rolled outputs use a `tags` array instead; - # accept both shapes so the count is robust to schema drift. - ftype = f.get("type") - tags = f.get("tags") or [] - is_sem = ( - (isinstance(ftype, str) and "semantic_duplication" in ftype) - or (isinstance(tags, list) - and any(isinstance(t, str) and "semantic_duplication" in t for t in tags)) - ) - if is_sem: - duplications.append(f) - # Set the field whenever either source provided ANY signal — including - # the explicit "structured zero" case (empty list with both sources - # checked). Without saw_any_source, an empty result here would drop the - # field entirely and let the share viewer fall through to its prose - # heuristic, which can return non-zero from unrelated scan_report text. - if saw_any_source: - bundle["semantic_duplications"] = duplications + # Including the explicit "structured zero" (empty list): without it the + # share viewer falls through to its prose heuristic, which can return + # non-zero from unrelated report text. + bundle["semantic_duplications"] = sem["duplications"] return bundle diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 81ff6956..49ec1456 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -28,7 +28,7 @@ Comprehensive technical documentation covering system architecture, analysis pip 20. [StructuredBlueprint Data Model](#structuredblueprint-data-model) 21. [Data Flow](#data-flow) 22. [Compound Learning](#compound-learning) -23. [Drift Detection](#drift-detection) +23. [Drift Coverage](#drift-coverage-no-dedicated-drift-step) 24. [Cycle Detection](#cycle-detection) 25. [Telemetry](#telemetry) 26. [No Inline Python Constraint](#no-inline-python-constraint) @@ -126,7 +126,6 @@ archie/ renderer.py # Generate AGENTS.md (canonical) + CLAUDE.md pointer + .claude/rules/ topic files intent_layer.py # Per-folder CLAUDE.md via DAG scheduling + AI enrichment + inspect/scan-config/deep-scan-state/save-run-context viewer.py # Local viewer — serves the React dist/ + /api/* JSON endpoints (stdlib http.server) - drift.py # Mechanical drift detection validate.py # Cross-reference blueprint against actual codebase check_rules.py # Check files against rules (CI path) measure_health.py # Erosion, gini, verbosity, top-20%, waste scores + history append + --compare-history @@ -142,7 +141,7 @@ archie/ migrate_blueprint_rules.py # Migrate legacy blueprint rule sections into proposed_rules.json arch_review.py # Architectural review checklist for plans and diffs refresh.py # File change detection (hash comparison) - extract_output.py # rules / deep-drift / recent-files / save-duplications subcommands + extract_output.py # rules / save-duplications subcommands telemetry.py # Per-run step-level wall-clock timing + steps-count action telemetry_sync.py # Anonymous opt-in telemetry — record events, push to Supabase update_check.py # Anonymous opt-in npm-registry update check + snooze ladder @@ -228,7 +227,7 @@ Claude Code slash commands (archie-deep-scan, archie-share, archie-viewer) v Standalone scripts (archie/standalone/*.py) <-- primary runtime path | scanner, measure_health, detect_cycles, - | finalize, merge, intent_layer, drift, + | finalize, merge, intent_layer, | extract_output, telemetry, upload, ... v File system + Claude Code subagent spawning (Agent tool) + Anthropic API @@ -243,7 +242,7 @@ A parallel Python-package path exists (`archie/cli/ + engine/ + coordinator/`) f - **Hooks** — real-time Claude Code integration. Registered in `.claude/settings.local.json` at install time. - **Renderer** — deterministic file generation. Input: `blueprint.json`. Output: CLAUDE.md, AGENTS.md, per-folder context, rule files. - **Rules** — extraction and severity management. Input: blueprint + AI proposals + platform rules. Output: `rules.json`. -- **Share** — `upload.py` builds a bundle (blueprint + findings + scan_report + health + rules) and POSTs it to the Supabase edge function; the React viewer renders it from a token URL. +- **Share** — `upload.py` builds a bundle (blueprint + findings + health + rules) and POSTs it to the Supabase edge function; the React viewer renders it from a token URL. --- @@ -376,9 +375,8 @@ Kept for CI/tests and standalone Python usage. Groups files into token-budgeted **Workflow structure.** `/archie-deep-scan` is a modular workflow tree, not a monolithic command file. The command shim is a thin router; the real pipeline is the rendered `deep-scan/` tree under `.archie/workflow//` — authored once as a template under `archie/assets/workflow/deep-scan/`: - `SKILL.md` — the orchestrator: flag parsing (`--incremental` / `--continue` / `--from N` / `--reconfigure`), the resume preamble, and a step-routing table. -- `steps/step-1-scanner.md` … `step-10-telemetry.md` — one self-contained file per step (Step 3 expands into `step-3-wave1/` with one prompt file per Wave 1 agent + shared grounding rules). +- `steps/step-1-scanner.md` … `step-9-finalize.md` — one self-contained file per step (Step 3 expands into `step-3-wave1/` with one prompt file per Wave 1 agent + shared grounding rules). - `fragments/` — cross-step contracts: `telemetry-conventions.md`, `compact-resume-contract.md`, `resume-prelude.md`. -- `templates/scan-report.md` — the scan-report template Step 9 fills in. - Phase 0 scope resolution is loaded from the shared `_shared/scope_resolution.md` fragment. The router Reads only the files for the steps it actually runs, so a `--from 7` resume never loads Steps 1–6, and `.archie/deep_scan_state.json` rehydrates shell variables after a `/compact`. @@ -407,10 +405,8 @@ Step 7 Intent Layer (opt-in) — per-folder CLAUDE.md via DAG scheduling If INTENT_LAYER=no from Step E, this step is skipped and telemetry records "skipped": true Step 8 Cleanup -Step 9 Drift Detection & Architectural Assessment - Mechanical drift (drift.py) + Deep AI drift (single Sonnet) - Writes scan_report.md to scan_history/ -Step 10 Telemetry write +Step 9 Finalize — health metrics (measure_health.py + history), + incremental baseline marker, telemetry write, closing summary ``` ### Incremental mode (`--incremental`) @@ -560,7 +556,7 @@ All hooks fail open: missing rules/config/marker files → hooks exit 0 silently Every analysis subagent spawned during a scan receives a mandatory instruction (the connector-rendered `{{>output_contract}}` partial) to write its complete output directly to `.archie/tmp/archie_*.json` — Claude renders this as a `Write` tool call, Codex renders it as `apply_patch` against the workspace path. Both land covered by `Write(.archie/**)` (Claude's permission allowlist) and Codex's default `workspace-write` sandbox. The orchestrator never copies subagent transcripts. This avoids Claude Code's sensitive-file guardrail on `~/.claude/projects/.../subagents/*.jsonl` (which used to fire a permission prompt on every batch), keeps subagent output out of the orchestrator's context (less compaction pressure), and isolates failures (missing confirmation line or missing file → clear signal, no silent fallback to transcript scraping). Artifacts are workspace-relative under `.archie/tmp/`, gitignored at install time via a self-ignoring `.archie/tmp/.gitignore` so they never get committed. -The contract is enforced in 6 spawn sites across the slash commands: Wave 1 structural agents (3–4 Sonnets), Wave 2 reasoning agent (full + incremental paths), rule-proposer agent, deep-drift reviewer, and Intent Layer enrichment subagents. +The contract is enforced in 5 spawn sites across the slash commands: Wave 1 structural agents (3–4 Sonnets), Wave 2 reasoning agents (full + incremental paths), rule-proposer agent, and Intent Layer enrichment subagents. --- @@ -676,7 +672,6 @@ Zero-dependency Python scripts in `archie/standalone/`. These are exported to ta | `renderer.py` | Blueprint JSON → AGENTS.md (canonical) + CLAUDE.md pointer + `.claude/rules/` topic files + `enforcement/` directory | | `intent_layer.py` | Per-folder CLAUDE.md via DAG scheduling + AI enrichment. Subcommands: `prepare`, `next-ready`, `suggest-batches`, `prompt`, `save-enrichment`, `merge`, `inspect [--query] [--list]`, `scan-config`, `deep-scan-state` (incl. `save-run-context` for shell-friendly run-context writes) | | `viewer.py` | Local viewer — stdlib `http.server` serving the React `dist/` + `/api/bundle` (reuses `upload.py::build_bundle`), `/api/generated-files`, `/api/folder-claude-mds`, `/api/intent-layer-status`, `/api/ignored-rules`, `POST /api/rules` (5 atomic rule actions); auto-reloads when its own source changes | -| `drift.py` | Mechanical drift detection | | `validate.py` | Cross-reference blueprint against actual codebase | | `check_rules.py` | Check files against rules (CI path) | | `measure_health.py` | Erosion, gini, verbosity, top-20%, waste scores + `--append-history` + `--compare-history` (trend deltas) | @@ -976,8 +971,7 @@ Local project Supabase (upload function) Supabas | - health.json (stripped) | | | - scan_meta, rules_adopted | | | - rules_proposed | | - | - scan_report.md | | - | - semantic_duplications | | + | - semantic_duplications (legacy) | | | v | |<---------------------------- {"token": "…"} | | | @@ -1064,8 +1058,7 @@ Shared across all three modes: "scan_meta": {...}, # frameworks, subproject count, frontend_ratio "rules_adopted": {...}, "rules_proposed": {...}, - "scan_report": "...", - "semantic_duplications": [...], # structured, from semantic_duplications.json + "semantic_duplications": [...], # legacy, from semantic_duplications.json when present "findings": [...] # from findings.json (shared store) } ``` @@ -1213,10 +1206,10 @@ Step 6 Rule synthesis — single Sonnet Step 7 Intent Layer — per-folder CLAUDE.md (OPT-IN, Step E decides) If INTENT_LAYER=no: skip, telemetry records "skipped": true Step 8 Cleanup .archie/tmp/archie_* -Step 9 Drift Detection & Architectural Assessment - drift.py (mechanical) + single Sonnet (deep AI drift) - Writes final scan_report.md + scan_history/scan_NNN_*.md -Step 10 Telemetry write to .archie/telemetry/deep-scan_.json +Step 9 Finalize + measure_health.py -> health.json + health_history.json + complete-step + save-baseline (incremental change detection) + Telemetry write to .archie/telemetry/deep-scan_.json ``` ### Incremental deep scan (`--incremental`) @@ -1247,33 +1240,20 @@ Every run feeds the next. Both commands read and write the same `findings.json` --- -## Drift Detection +## Drift Coverage (no dedicated drift step) -Deep scans include two-phase drift detection: +The standalone drift step was retired: its findings duplicated the Wave 2 Risk pipeline without the verifier/hysteresis guarantees, and its run time scaled with recent churn. The problem classes it covered now live in verified channels: -### Phase 1: Mechanical drift (`drift.py`) - -Deterministic analysis: -- Pattern outliers (files that don't match established patterns) -- File size and complexity violations -- Dependency-direction breaches -- Structural anomalies - -### Phase 2: Deep AI drift - -An agent reads blueprint + drift report + CLAUDE.md files + `findings.json` to identify drift categories: - -| Category | What it detects | +| Class | Where it's caught now | |----------|----------------| -| `decision_violation` | Code that contradicts a recorded architectural decision | -| `pattern_erosion` | Gradual drift away from established patterns | -| `trade_off_undermined` | Changes that undermine accepted trade-offs | -| `pitfall_triggered` | Known pitfalls that have materialised in code | -| `responsibility_leak` | Logic placed in the wrong component/layer | -| `abstraction_bypass` | Direct access that skips established abstractions | -| `semantic_duplication` | Reimplementation of existing functionality | +| `decision_violation` | Risk agent's invariant walk (Step 5b) + `pre-validate.sh` blocks at edit time | +| `pattern_erosion` | Incremental recency sweep (Risk agent reads changed files against per-folder CLAUDE.md) | +| `trade_off_undermined` | Risk agent reads `trade_offs.violation_signals`; hooks warn via `tradeoff_undermined` | +| `pitfall_triggered` | Risk agent pitfall pass + hooks block via `pitfall_triggered` | +| `responsibility_leak` / `abstraction_bypass` | Risk agent's whole-system pass (cross-component coupling, decision-chain constraints) | +| schema drift | Risk agent's data-shaped pitfall classes (`data_models` + `persistence_stores`) | -Violations are grounded with `violation_signals` from the blueprint's trade-off and decision chain data. +Everything user-facing flows through `findings.json` — `triggering_call_site` required, backward-verified, hysteresis-stabilised. --- @@ -1308,7 +1288,7 @@ Every `/archie-deep-scan` run writes a per-step wall-clock timing file: {"name": "rule_synthesis", "seconds": 316, "model": "sonnet"}, {"name": "intent_layer", "seconds": 19676, "model": "sonnet", "skipped": false}, {"name": "cleanup", "seconds": 22}, - {"name": "drift", "seconds": 340} + {"name": "finalize", "seconds": 18} ] } ``` @@ -1346,8 +1326,6 @@ Scan templates include a "CRITICAL CONSTRAINT: Never write inline Python" block | `intent_layer.py scan-config read\|write\|validate` | Monorepo scope config management | | `intent_layer.py deep-scan-state init\|read\|complete-step N\|detect-changes\|save-baseline\|save-context\|save-run-context` | Deep-scan resume state; `save-run-context` takes flags + newline-separated workspaces via stdin (replaces heredoc-JSON-with-inline-python that used to round-trip workspaces through `python3 -c`) | | `extract_output.py rules ` | Parse rule synthesis output | -| `extract_output.py deep-drift ` | Merge drift findings into report | -| `extract_output.py recent-files ` | Print source file paths | | `extract_output.py save-duplications ` | Deterministic semantic_duplications.json writer | | `telemetry.py --command --timing-file ` | Write per-run telemetry | | `telemetry.py steps-count ` | Count completed step marks (replaces inline python `len(json.load(...).get('steps'))` in deep-scan Resume Prelude) | diff --git a/npm-package/README.md b/npm-package/README.md index 073b1537..ac31ba89 100644 --- a/npm-package/README.md +++ b/npm-package/README.md @@ -25,7 +25,7 @@ Then open your project in Claude Code and run: - Installs 4 real-time enforcement hooks (file validation, commit review, plan review, architecture nudge) - Proposes enforceable rules with confidence scores and deep rationale - Tracks health metrics over time (erosion, complexity distribution, duplication) -- Detects architectural drift and dependency cycles +- Surfaces verified architectural findings and dependency cycles - Ships 40+ platform-specific rules (Android, Swift, TypeScript, Python) ## What it generates diff --git a/npm-package/assets/_common.py b/npm-package/assets/_common.py index 5c673c1b..6a7c0c3c 100644 --- a/npm-package/assets/_common.py +++ b/npm-package/assets/_common.py @@ -240,7 +240,7 @@ def is_ignored(self, rel_path: str) -> bool: Gitignore semantics: a path is also ignored when any ANCESTOR directory matches a dir pattern (``vendor/`` ignores ``vendor/b.py``). os.walk callers get this for free via directory pruning; full-path callers - (e.g. drift's git-log list, where the file may no longer exist on disk) + (e.g. a git-log file list, where the file may no longer exist on disk) rely on this ancestor walk. """ rel_path = rel_path.replace(os.sep, "/") diff --git a/npm-package/assets/_install_pkg/install.py b/npm-package/assets/_install_pkg/install.py index 63c3296e..7c19ec86 100644 --- a/npm-package/assets/_install_pkg/install.py +++ b/npm-package/assets/_install_pkg/install.py @@ -43,7 +43,7 @@ def _resolve_targets(requested: list[str] | None, connectors: list[Connector]) - # Analysis pipeline (referenced by SKILL bodies via `python3 .archie/`) "scanner.py", "renderer.py", "validate.py", "intent_layer.py", "finalize.py", "merge.py", "measure_health.py", "detect_cycles.py", - "drift.py", "extract_output.py", "arch_review.py", "align_check.py", + "extract_output.py", "arch_review.py", "align_check.py", "check_rules.py", "code_shape.py", "rule_index.py", "lint_gate.py", "agent_cli.py", "verify_findings.py", "apply_verdicts.py", "migrate_blueprint_rules.py", "rule_kinds.py", "backfill_kinds.py", @@ -87,6 +87,16 @@ def _clean_legacy_layout(project_root: Path) -> None: except OSError: pass + # Retired pipeline scripts — remove on upgrade so stale copies don't linger + # in .archie/. (The npm installer wipes all .py files before copying; this + # pip path copies over without sweeping, so retired names need an explicit + # delete.) + for retired in ("drift.py",): + try: + (project_root / ".archie" / retired).unlink() + except OSError: + pass + current_command_names = {c.name for c in COMMANDS} # Sweep stale Claude command shims (.claude/commands/archie-X.md). diff --git a/npm-package/assets/_install_pkg/manifest_data.py b/npm-package/assets/_install_pkg/manifest_data.py index 86788306..c070fcc5 100644 --- a/npm-package/assets/_install_pkg/manifest_data.py +++ b/npm-package/assets/_install_pkg/manifest_data.py @@ -126,17 +126,18 @@ claude_glob="Bash(python3 -c *)", justification="Inline Python — Archie workflow uses for JSON inspection", ), - # Codex spawning Codex for the headless finding-verifier in Step 9 drift. - # verify_findings.py shells out to `codex exec --sandbox read-only …` - # (see archie/standalone/agent_cli.py::_run_codex). Without this rule the - # parent Codex session would prompt mid-Step-9 on every verified finding. + # Codex spawning Codex for the headless finding-verifier (deep-scan Step 5, + # after finalize writes findings.json). verify_findings.py shells out to + # `codex exec --sandbox read-only …` (see agent_cli.py::_run_codex). + # Without this rule the parent Codex session would prompt mid-Step-5 on + # every verified finding. CommandRule( name="codex-exec", codex_pattern=("codex", "exec"), claude_glob="Bash(codex exec *)", justification=( "verify_findings.py spawns `codex exec` for per-finding model " - "calls during Step 9 drift verification" + "calls during Step 5 finding verification" ), ), # Filesystem prep / inspection diff --git a/npm-package/assets/drift.py b/npm-package/assets/drift.py deleted file mode 100644 index 2d6bc9d0..00000000 --- a/npm-package/assets/drift.py +++ /dev/null @@ -1,722 +0,0 @@ -#!/usr/bin/env python3 -"""Archie drift detector — finds architectural divergences and outliers. - -Compares per-folder enrichments and file structure against the blueprint's -dominant patterns to surface: pattern inconsistencies, naming violations, -dependency direction breaches, and structural outliers. - -Run: - python3 drift.py /path/to/repo - -Output: JSON drift report to stdout, human summary to stderr. - -Zero dependencies beyond Python 3.9+ stdlib. -""" -from __future__ import annotations - -import json -import re -import sys -from collections import Counter, defaultdict -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -from _common import _load_json # noqa: E402 - - -# --------------------------------------------------------------------------- -# 1. Pattern consistency — which folders diverge from dominant patterns -# --------------------------------------------------------------------------- - -def check_pattern_consistency(root: Path) -> list[dict]: - """Find folders whose patterns diverge from the codebase-wide norms.""" - enrichments_dir = root / ".archie" / "enrichments" - if not enrichments_dir.is_dir(): - return [] - - # Collect all patterns across all folders - folder_patterns: dict[str, list[str]] = {} - all_enrichments: dict[str, dict] = {} - for json_file in sorted(enrichments_dir.iterdir()): - if not json_file.name.endswith(".json"): - continue - try: - data = json.loads(json_file.read_text()) - if isinstance(data, dict): - all_enrichments.update(data) - except (json.JSONDecodeError, OSError): - continue - - if not all_enrichments: - return [] - - # Extract pattern names per folder - pattern_counter: Counter = Counter() - for folder, info in all_enrichments.items(): - if not isinstance(info, dict): - continue - patterns = info.get("patterns", []) - names = [] - for p in patterns: - if isinstance(p, dict) and p.get("name"): - names.append(p["name"].lower().strip()) - elif isinstance(p, str): - names.append(p.lower().strip()) - folder_patterns[folder] = names - for n in names: - pattern_counter[n] += 1 - - if not pattern_counter: - return [] - - # Find dominant patterns (used in >25% of folders) - threshold = max(2, len(folder_patterns) * 0.25) - dominant = {name for name, count in pattern_counter.items() if count >= threshold} - - # Group folders by their parent to find sibling divergences - siblings: dict[str, list[str]] = defaultdict(list) - for folder in folder_patterns: - parent = str(Path(folder).parent) - siblings[parent].append(folder) - - findings = [] - - # Check sibling consistency — folders under the same parent should use - # similar patterns - for parent, sibs in siblings.items(): - if len(sibs) < 2: - continue - # Collect patterns used by siblings - sib_patterns: Counter = Counter() - for s in sibs: - for p in folder_patterns.get(s, []): - sib_patterns[p] += 1 - # Sibling-dominant: used by >50% of siblings - sib_threshold = len(sibs) * 0.5 - sib_dominant = {name for name, count in sib_patterns.items() if count >= sib_threshold} - - for s in sibs: - s_pats = set(folder_patterns.get(s, [])) - missing = sib_dominant - s_pats - if missing and s_pats: # has some patterns but missing dominant ones - findings.append({ - "type": "pattern_divergence", - "folder": s, - "message": f"Siblings under {parent}/ mostly use [{', '.join(sorted(missing))}] but this folder does not", - "expected": sorted(sib_dominant), - "actual": sorted(s_pats), - "severity": "warn", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 2. Naming convention violations -# --------------------------------------------------------------------------- - -def check_naming_conventions(root: Path) -> list[dict]: - """Check files against naming conventions from the blueprint.""" - bp = _load_json(root / ".archie" / "blueprint.json") - rules = bp.get("architecture_rules", {}) - conventions = rules.get("naming_conventions", []) - scan = _load_json(root / ".archie" / "scan.json") - files = [f.get("path", "") for f in scan.get("file_tree", [])] - - if not conventions or not files: - return [] - - findings = [] - for conv in conventions: - if not isinstance(conv, dict): - continue - scope = conv.get("scope", "") - pattern = conv.get("pattern", "") - description = conv.get("description", "") - examples = conv.get("examples", []) - - if not pattern or not scope: - continue - - # Try to build a regex from the pattern description - # Common patterns: PascalCase, camelCase, snake_case, kebab-case - regex = None - pat_lower = pattern.lower() - if "pascalcase" in pat_lower or "pascal case" in pat_lower: - regex = r'^[A-Z][a-zA-Z0-9]*' - elif "camelcase" in pat_lower or "camel case" in pat_lower: - regex = r'^[a-z][a-zA-Z0-9]*' - elif "snake_case" in pat_lower or "snake case" in pat_lower: - regex = r'^[a-z][a-z0-9_]*' - elif "kebab" in pat_lower: - regex = r'^[a-z][a-z0-9\-]*' - - if not regex: - continue - - # Find files that match the scope but violate the pattern - violations = [] - scope_lower = scope.lower() - for fp in files: - # Check if file is in scope — match path segments, not substrings - fp_lower = fp.lower() - if not (fp_lower.startswith(scope_lower + "/") or - ("/" + scope_lower + "/") in fp_lower or - fp_lower == scope_lower): - continue - filename = fp.rsplit("/", 1)[-1] if "/" in fp else fp - name_part = filename.rsplit(".", 1)[0] if "." in filename else filename - if not re.match(regex, name_part): - violations.append(fp) - - if violations and len(violations) <= len(files) * 0.3: - # Only report if it's a minority (actual outliers, not a bad rule) - findings.append({ - "type": "naming_violation", - "convention": description or pattern, - "scope": scope, - "violating_files": violations if _COMPREHENSIVE else violations[:10], - "count": len(violations), - "severity": "info", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 3. Dependency direction violations -# --------------------------------------------------------------------------- - -def check_dependency_direction(root: Path) -> list[dict]: - """Check import graph against component dependency declarations.""" - bp = _load_json(root / ".archie" / "blueprint.json") - scan = _load_json(root / ".archie" / "scan.json") - import_graph = scan.get("import_graph", {}) - - comps_raw = bp.get("components", {}) - components = comps_raw.get("components", []) if isinstance(comps_raw, dict) else comps_raw if isinstance(comps_raw, list) else [] - - if not import_graph or not components: - return [] - - # Build component location map and allowed dependency map - comp_by_loc: dict[str, dict] = {} - for comp in components: - loc = (comp.get("location") or "").rstrip("/") - if loc: - comp_by_loc[loc] = comp - - findings = [] - - # For each component, check if files import from components NOT in depends_on - for comp in components: - loc = (comp.get("location") or "").rstrip("/") - if not loc: - continue - allowed_deps = set(comp.get("depends_on", [])) - comp_name = comp.get("name", loc) - - # Find all imports from files in this component - for file_path, imports in import_graph.items(): - if not (file_path.startswith(loc + "/") or file_path == loc): - continue - for imp in imports: - imp_path = imp.replace(".", "/") - # Check if this import lands in another component - for other_loc, other_comp in comp_by_loc.items(): - if other_loc == loc: - continue - other_name = other_comp.get("name", other_loc) - if imp_path.startswith(other_loc) or other_loc in imp_path: - if other_name not in allowed_deps and other_loc not in allowed_deps: - findings.append({ - "type": "dependency_violation", - "from_component": comp_name, - "to_component": other_name, - "file": file_path, - "import": imp, - "message": f"{comp_name} imports from {other_name} but does not declare it as a dependency", - "severity": "warn", - }) - - # Deduplicate by (from, to) pair — keep first occurrence - seen = set() - deduped = [] - for f in findings: - key = (f["from_component"], f["to_component"]) - if key not in seen: - seen.add(key) - deduped.append(f) - - return deduped - - -# --------------------------------------------------------------------------- -# 4. Structural outliers — folders organized differently from siblings -# --------------------------------------------------------------------------- - -def check_structural_outliers(root: Path) -> list[dict]: - """Find folders that are structurally different from their siblings.""" - scan = _load_json(root / ".archie" / "scan.json") - files = scan.get("file_tree", []) - - if not files: - return [] - - # Count files per directory and collect extensions - dir_file_counts: dict[str, int] = defaultdict(int) - dir_extensions: dict[str, Counter] = defaultdict(Counter) - for f in files: - p = f.get("path", "") - if "/" not in p: - continue - parent = str(Path(p).parent) - dir_file_counts[parent] += 1 - ext = f.get("extension", "") - if ext: - dir_extensions[parent][ext] += 1 - - # Group by grandparent to find sibling directories - siblings: dict[str, list[str]] = defaultdict(list) - for d in dir_file_counts: - grandparent = str(Path(d).parent) - siblings[grandparent].append(d) - - findings = [] - for grandparent, sibs in siblings.items(): - if len(sibs) < 3: - continue - - # Check for file count outliers (>3 standard deviations) - counts = [dir_file_counts[s] for s in sibs] - if not counts: - continue - avg = sum(counts) / len(counts) - if avg == 0: - continue - variance = sum((c - avg) ** 2 for c in counts) / len(counts) - std = variance ** 0.5 - - if std == 0: - continue - - for s in sibs: - count = dir_file_counts[s] - if std > 0 and abs(count - avg) > 3 * std and count > avg: - findings.append({ - "type": "structural_outlier", - "folder": s, - "message": f"Has {count} files — significantly more than sibling average ({avg:.0f}). May be a god-folder that should be split.", - "severity": "info", - }) - - # Check for extension mismatches — siblings should have similar file types - sib_exts: Counter = Counter() - for s in sibs: - for ext in dir_extensions[s]: - sib_exts[ext] += 1 - # Dominant extension: used by >60% of siblings - dominant_exts = {ext for ext, cnt in sib_exts.items() if cnt >= len(sibs) * 0.6} - - for s in sibs: - s_exts = set(dir_extensions[s].keys()) - unexpected = s_exts - dominant_exts - if unexpected and dominant_exts and len(unexpected) > len(dominant_exts): - findings.append({ - "type": "structural_outlier", - "folder": s, - "message": f"Uses [{', '.join(sorted(unexpected))}] while siblings mostly use [{', '.join(sorted(dominant_exts))}]", - "severity": "info", - }) - - return findings - - -# --------------------------------------------------------------------------- -# 5. Anti-pattern clusters — folders with many anti-patterns -# --------------------------------------------------------------------------- - -def check_antipattern_clusters(root: Path) -> list[dict]: - """Find folders with an unusual density of anti-patterns.""" - enrichments_dir = root / ".archie" / "enrichments" - if not enrichments_dir.is_dir(): - return [] - - all_enrichments: dict[str, dict] = {} - for json_file in sorted(enrichments_dir.iterdir()): - if not json_file.name.endswith(".json"): - continue - try: - data = json.loads(json_file.read_text()) - if isinstance(data, dict): - all_enrichments.update(data) - except (json.JSONDecodeError, OSError): - continue - - if not all_enrichments: - return [] - - # Count anti-patterns per folder - ap_counts = {} - for folder, info in all_enrichments.items(): - if not isinstance(info, dict): - continue - anti = info.get("anti_patterns", []) - if anti: - ap_counts[folder] = len(anti) - - if not ap_counts: - return [] - - avg = sum(ap_counts.values()) / len(ap_counts) if ap_counts else 0 - threshold = max(avg * 2, 4) - - findings = [] - for folder, count in sorted(ap_counts.items(), key=lambda x: -x[1]): - if count >= threshold: - anti = all_enrichments[folder].get("anti_patterns", []) - findings.append({ - "type": "antipattern_cluster", - "folder": folder, - "count": count, - "anti_patterns": anti if _COMPREHENSIVE else anti[:5], - "message": f"Has {count} anti-patterns (avg: {avg:.1f}) — high-risk area for accidental violations", - "severity": "warn", - }) - - return findings - - -# --------------------------------------------------------------------------- -# Main report -# --------------------------------------------------------------------------- - -_SECTIONS = [ - ("pattern_divergences", "Pattern Divergences"), - ("dependency_violations", "Dependency Violations"), - ("naming_violations", "Naming Violations"), - ("structural_outliers", "Structural Outliers"), - ("antipattern_clusters", "Anti-Pattern Clusters"), -] - - -def generate_drift_report(root: Path) -> dict: - """Generate a full drift report.""" - from datetime import datetime, timezone - - report = { - "pattern_divergences": check_pattern_consistency(root), - "naming_violations": check_naming_conventions(root), - "dependency_violations": check_dependency_direction(root), - "structural_outliers": check_structural_outliers(root), - "antipattern_clusters": check_antipattern_clusters(root), - } - - # Summary counts - total = sum(len(v) for v in report.values()) - warns = sum(1 for v in report.values() for f in v if f.get("severity") == "warn") - infos = total - warns - - report["summary"] = { - "total_findings": total, - "warnings": warns, - "informational": infos, - "checks_run": [k for k, v in report.items() if k != "summary"], - "timestamp": datetime.now(timezone.utc).isoformat(), - } - - return report - - -def print_human_summary(report: dict): - """Print human-readable drift summary to stderr.""" - summary = report.get("summary", {}) - total = summary.get("total_findings", 0) - - print(f"\nDrift Analysis: {total} findings", file=sys.stderr) - - for section, label in _SECTIONS: - items = report.get(section, []) - if not items: - continue - print(f"\n {label} ({len(items)}):", file=sys.stderr) - for item in items[:5]: - sev = item.get("severity", "info").upper() - msg = item.get("message", "") - folder = item.get("folder", "") - if folder: - print(f" [{sev}] {folder}: {msg}", file=sys.stderr) - else: - print(f" [{sev}] {msg}", file=sys.stderr) - if len(items) > 5: - print(f" ... and {len(items) - 5} more", file=sys.stderr) - - if total == 0: - print(" No architectural drift detected.", file=sys.stderr) - print("", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# Snapshot history -# --------------------------------------------------------------------------- - -_HISTORY_DIR = "drift_history" - - -def _save_snapshot(root: Path, report: dict): - """Save a timestamped drift snapshot for future diffing.""" - from datetime import datetime, timezone - - history_dir = root / ".archie" / _HISTORY_DIR - history_dir.mkdir(parents=True, exist_ok=True) - - ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - snapshot_path = history_dir / f"drift_{ts}.json" - snapshot_path.write_text(json.dumps(report, indent=2)) - - # Also update the "latest" symlink / copy for easy access - latest_path = history_dir / "latest.json" - latest_path.write_text(json.dumps(report, indent=2)) - - return snapshot_path - - -def _load_previous_snapshot(root: Path) -> dict | None: - """Load the most recent snapshot before the current one.""" - history_dir = root / ".archie" / _HISTORY_DIR - if not history_dir.is_dir(): - return None - - snapshots = sorted( - [f for f in history_dir.iterdir() if f.name.startswith("drift_") and f.name.endswith(".json")], - key=lambda f: f.name, - ) - - # Return second-to-last (the previous run), since the current run - # may have already been saved - if len(snapshots) >= 2: - try: - return json.loads(snapshots[-2].read_text()) - except (json.JSONDecodeError, OSError): - pass - return None - - -# --------------------------------------------------------------------------- -# Diff engine -# --------------------------------------------------------------------------- - -def _finding_key(finding: dict) -> str: - """Create a stable identity key for a finding so we can diff across runs.""" - ftype = finding.get("type", "") - folder = finding.get("folder", "") - # For dependency violations, include both ends - from_comp = finding.get("from_component", "") - to_comp = finding.get("to_component", "") - # For naming violations, include scope - scope = finding.get("scope", "") - convention = finding.get("convention", "") - # Include message prefix to disambiguate same-folder findings - # (e.g., two structural_outlier findings for the same folder: file count vs extension) - msg = finding.get("message", "")[:80] - - parts = [ftype, folder, from_comp, to_comp, scope, convention, msg] - return "|".join(p for p in parts if p) - - -def compute_diff(previous: dict, current: dict) -> dict: - """Compare two drift reports and return new, resolved, and persisting findings.""" - diff_result = { - "new": [], # in current but not in previous - "resolved": [], # in previous but not in current - "persisting": [], # in both - } - - prev_ts = previous.get("summary", {}).get("timestamp", "unknown") - curr_ts = current.get("summary", {}).get("timestamp", "unknown") - - # Collect all findings from both reports, keyed - prev_findings: dict[str, dict] = {} - curr_findings: dict[str, dict] = {} - - for section, _ in _SECTIONS: - for f in previous.get(section, []): - prev_findings[_finding_key(f)] = f - for f in current.get(section, []): - curr_findings[_finding_key(f)] = f - - prev_keys = set(prev_findings.keys()) - curr_keys = set(curr_findings.keys()) - - for key in sorted(curr_keys - prev_keys): - diff_result["new"].append(curr_findings[key]) - for key in sorted(prev_keys - curr_keys): - diff_result["resolved"].append(prev_findings[key]) - for key in sorted(curr_keys & prev_keys): - diff_result["persisting"].append(curr_findings[key]) - - diff_result["summary"] = { - "previous_timestamp": prev_ts, - "current_timestamp": curr_ts, - "previous_total": previous.get("summary", {}).get("total_findings", 0), - "current_total": current.get("summary", {}).get("total_findings", 0), - "new_findings": len(diff_result["new"]), - "resolved_findings": len(diff_result["resolved"]), - "persisting_findings": len(diff_result["persisting"]), - } - - return diff_result - - -def print_diff_summary(diff_result: dict): - """Print human-readable diff to stderr.""" - s = diff_result["summary"] - prev_total = s["previous_total"] - curr_total = s["current_total"] - delta = curr_total - prev_total - - delta_str = f"+{delta}" if delta > 0 else str(delta) - print(f"\nDrift Diff: {prev_total} -> {curr_total} ({delta_str})", file=sys.stderr) - print(f" Previous: {s['previous_timestamp']}", file=sys.stderr) - print(f" Current: {s['current_timestamp']}", file=sys.stderr) - - new = diff_result["new"] - resolved = diff_result["resolved"] - persisting = diff_result["persisting"] - - if new: - print(f"\n NEW ({len(new)}):", file=sys.stderr) - for item in new: - sev = item.get("severity", "info").upper() - folder = item.get("folder", "") - msg = item.get("message", "") - print(f" + [{sev}] {folder}: {msg}", file=sys.stderr) - - if resolved: - print(f"\n RESOLVED ({len(resolved)}):", file=sys.stderr) - for item in resolved: - sev = item.get("severity", "info").upper() - folder = item.get("folder", "") - msg = item.get("message", "") - print(f" - [{sev}] {folder}: {msg}", file=sys.stderr) - - if persisting: - print(f"\n PERSISTING ({len(persisting)})", file=sys.stderr) - - if not new and not resolved: - print("\n No changes since last run.", file=sys.stderr) - - print("", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def cmd_run(root: Path): - """Run drift detection, save snapshot, auto-diff against previous, print summary.""" - report = generate_drift_report(root) - print_human_summary(report) - - # Save report and snapshot - out_path = root / ".archie" / "drift_report.json" - out_path.write_text(json.dumps(report, indent=2)) - snapshot_path = _save_snapshot(root, report) - print(f"Saved: {out_path}", file=sys.stderr) - print(f"Snapshot: {snapshot_path}", file=sys.stderr) - - # Auto-diff against previous snapshot if one exists - previous = _load_previous_snapshot(root) - if previous: - diff_result = compute_diff(previous, report) - print_diff_summary(diff_result) - - diff_path = root / ".archie" / "drift_diff.json" - diff_path.write_text(json.dumps(diff_result, indent=2)) - - # Output JSON to stdout - print(json.dumps(report, indent=2)) - - -def cmd_history(root: Path): - """List all drift snapshots with finding counts.""" - history_dir = root / ".archie" / _HISTORY_DIR - if not history_dir.is_dir(): - print("No drift history found.", file=sys.stderr) - return - - snapshots = sorted( - [f for f in history_dir.iterdir() if f.name.startswith("drift_") and f.name.endswith(".json")], - key=lambda f: f.name, - ) - - if not snapshots: - print("No drift snapshots found.", file=sys.stderr) - return - - entries = [] - prev_total = None - for snap in snapshots: - try: - data = json.loads(snap.read_text()) - s = data.get("summary", {}) - total = s.get("total_findings", 0) - warns = s.get("warnings", 0) - ts = s.get("timestamp", "") - - delta = "" - if prev_total is not None: - d = total - prev_total - delta = f" (+{d})" if d > 0 else f" ({d})" if d < 0 else " (=)" - prev_total = total - - entries.append({ - "file": snap.name, - "timestamp": ts, - "total": total, - "warnings": warns, - "delta": delta, - }) - except (json.JSONDecodeError, OSError): - continue - - print(f"\nDrift History ({len(entries)} snapshots):\n", file=sys.stderr) - for e in entries: - print(f" {e['file']} {e['total']:3d} findings ({e['warnings']} warn){e['delta']}", file=sys.stderr) - print("", file=sys.stderr) - - # JSON to stdout - print(json.dumps(entries, indent=2)) - - -# Comprehensive-depth switch: lift the report detail-list caps (full lists, no -# [:10]/[:5]). Read from run state; *count fields are unaffected. -_COMPREHENSIVE = False - - -def _is_comprehensive(root) -> bool: - try: - st = _load_json(Path(root) / ".archie" / "deep_scan_state.json") - return (st.get("run_context") or {}).get("depth") == "comprehensive" - except Exception: - return False - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage:", file=sys.stderr) - print(" python3 drift.py /path/to/repo — run drift detection (auto-diffs if previous exists)", file=sys.stderr) - print(" python3 drift.py history /path/to/repo — list all snapshots", file=sys.stderr) - sys.exit(1) - - # Parse subcommand - if sys.argv[1] == "history": - if len(sys.argv) < 3: - print("Usage: python3 drift.py history /path/to/repo", file=sys.stderr) - sys.exit(1) - root = Path(sys.argv[2]).resolve() - cmd_history(root) - else: - root = Path(sys.argv[1]).resolve() - _COMPREHENSIVE = _is_comprehensive(root) or ("--comprehensive" in sys.argv) - cmd_run(root) diff --git a/npm-package/assets/extract_output.py b/npm-package/assets/extract_output.py index d0db0906..89129e7e 100644 --- a/npm-package/assets/extract_output.py +++ b/npm-package/assets/extract_output.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 """Archie extract_output — robust extraction of agent output for the pipeline. -Replaces all inline python3 -c one-liners in archie-init.md and archie-drift.md. +Replaces inline python3 -c one-liners in the workflow files. Uses merge.extract_json_from_text to handle conversation envelopes, code fences, and AI escape issues. Subcommands: rules — extract rules JSON from agent output - deep-drift — extract deep findings, merge into drift report - recent-files — print source file paths from scan.json save-duplications — write .archie/semantic_duplications.json Zero dependencies beyond Python 3.9+ stdlib + sibling merge.py. @@ -127,68 +125,6 @@ def cmd_rules(input_file: str, output_path: str): print(f"Saved {len(new_rules)} rules to {output_path}", file=sys.stderr) -# --------------------------------------------------------------------------- -# deep-drift — extract deep findings and merge into drift report -# --------------------------------------------------------------------------- - -def cmd_deep_drift(input_file: str, report_path: str): - """Extract deep architectural findings from agent output, merge into drift report.""" - text = Path(input_file).read_text() - data = extract_json_from_text(text) - - if not data: - print("Warning: could not extract deep findings", file=sys.stderr) - sys.exit(1) - - report_file = Path(report_path) - if report_file.exists(): - report = json.loads(report_file.read_text()) - else: - report = {"summary": {"total_findings": 0, "warnings": 0}} - - deep_findings = data.get("deep_findings", []) - report["deep_findings"] = deep_findings - - s = report["summary"] - deep_count = len(deep_findings) - s["deep_findings"] = deep_count - s["total_findings"] = s.get("total_findings", 0) + deep_count - s["warnings"] = s.get("warnings", 0) + sum( - 1 for f in deep_findings if f.get("severity") == "warn" - ) - - report_file.write_text(json.dumps(report, indent=2)) - print(f"Added {deep_count} deep findings to {report_path}", file=sys.stderr) - - -# --------------------------------------------------------------------------- -# recent-files — print source file paths from scan.json -# --------------------------------------------------------------------------- - -_SOURCE_EXTENSIONS = { - ".kt", ".java", ".swift", ".ts", ".tsx", ".js", ".jsx", - ".py", ".go", ".rs", ".rb", ".dart", ".cs", ".cpp", ".c", ".h", - ".m", ".scala", ".clj", ".ex", ".exs", ".zig", ".lua", -} - - -def cmd_recent_files(scan_json: str): - """Print source file paths from scan.json, one per line.""" - data = json.loads(Path(scan_json).read_text()) - file_tree = data.get("file_tree", []) - - count = 0 - for f in file_tree: - ext = f.get("extension", "") - if ext in _SOURCE_EXTENSIONS: - print(f["path"]) - count += 1 - if count >= 100: - break - - print(f"Listed {count} source files", file=sys.stderr) - - # --------------------------------------------------------------------------- # save-duplications — extract Agent C's duplications and write to .archie/ # --------------------------------------------------------------------------- @@ -230,8 +166,6 @@ def cmd_save_duplications(agent_c_file: str, project_root: str): if len(sys.argv) < 2: print("Usage:", file=sys.stderr) print(" python3 extract_output.py rules ", file=sys.stderr) - print(" python3 extract_output.py deep-drift ", file=sys.stderr) - print(" python3 extract_output.py recent-files ", file=sys.stderr) print(" python3 extract_output.py save-duplications ", file=sys.stderr) sys.exit(1) @@ -243,18 +177,6 @@ def cmd_save_duplications(agent_c_file: str, project_root: str): sys.exit(1) cmd_rules(sys.argv[2], sys.argv[3]) - elif subcmd == "deep-drift": - if len(sys.argv) < 4: - print("Usage: extract_output.py deep-drift ", file=sys.stderr) - sys.exit(1) - cmd_deep_drift(sys.argv[2], sys.argv[3]) - - elif subcmd == "recent-files": - if len(sys.argv) < 3: - print("Usage: extract_output.py recent-files ", file=sys.stderr) - sys.exit(1) - cmd_recent_files(sys.argv[2]) - elif subcmd == "save-duplications": if len(sys.argv) < 4: print("Usage: extract_output.py save-duplications ", file=sys.stderr) diff --git a/npm-package/assets/intent_layer.py b/npm-package/assets/intent_layer.py index 2e6e774f..b45c84e6 100644 --- a/npm-package/assets/intent_layer.py +++ b/npm-package/assets/intent_layer.py @@ -595,7 +595,7 @@ def _load_state() -> dict: _STEP_NAMES = { 1: "scan", 2: "read", 3: "wave1", 4: "merge", 5: "wave2_synthesis", 6: "rule_synthesis", - 7: "intent_layer", 8: "cleanup", 9: "drift", + 7: "intent_layer", 8: "cleanup", 9: "finalize", } step_name = _STEP_NAMES.get(step) if step_name: @@ -2304,26 +2304,6 @@ def cmd_inspect(root: Path, filename: str, query: str | None = None, as_list: bo print(json.dumps(data, indent=2)) -def cmd_filter_ignored(root: Path): - """Read newline-separated repo-relative paths on stdin; print those NOT - ignored by .gitignore/.archieignore. Used by step-9 drift so its git-log - file list honors the ignore system (git does not know .archieignore). - - ``IgnoreMatcher.is_ignored`` applies full gitignore semantics, including - ancestor-directory matches (``vendor/`` hides ``vendor/b.py``).""" - matcher = IgnoreMatcher(root) - - def _is_path_ignored(rel: str) -> bool: - return matcher.is_ignored(rel.replace(os.sep, "/")) - - for line in sys.stdin: - rel = line.strip() - if not rel: - continue - if not _is_path_ignored(rel): - print(rel) - - # --------------------------------------------------------------------------- # CLI entry point # --------------------------------------------------------------------------- @@ -2456,8 +2436,6 @@ def _is_path_ignored(rel: str) -> bool: cmd_inject_scoped(root) elif subcmd == "extract-guardrails": cmd_extract_guardrails(root) - elif subcmd == "filter-ignored": - cmd_filter_ignored(Path(sys.argv[2]).resolve()) elif subcmd == "inspect": if len(sys.argv) < 4: print("Usage: inspect /path/to/repo [--query .key.path] [--list]", file=sys.stderr) diff --git a/npm-package/assets/sync.py b/npm-package/assets/sync.py index 8b1052f1..1de49f7c 100644 --- a/npm-package/assets/sync.py +++ b/npm-package/assets/sync.py @@ -333,7 +333,7 @@ def cmd_record(root: Path, input_file: str | None, agent: str, since: str | None "claims": out_claims, } - # 5. Write the versioned change file + latest.json (mirror drift._save_snapshot). + # 5. Write the versioned change file + latest.json (timestamped snapshot history). changes_dir = _changes_dir(root) changes_dir.mkdir(parents=True, exist_ok=True) record["version"] = _next_version(changes_dir) diff --git a/npm-package/assets/telemetry.py b/npm-package/assets/telemetry.py index 9f99959b..fe0f8307 100644 --- a/npm-package/assets/telemetry.py +++ b/npm-package/assets/telemetry.py @@ -293,7 +293,7 @@ def _detect_cli() -> str: "rule_synthesis": "Rule synthesis", "intent_layer": "Intent Layer", "cleanup": "Cleanup", - "drift": "Drift & assessment", + "finalize": "Finalize", } # Parallel sub-agent friendly labels (Wave 1 fact agents + Wave 2 reasoning agents). _AGENT_LABELS = { diff --git a/npm-package/assets/upload.py b/npm-package/assets/upload.py index d02261ef..1a052efa 100644 --- a/npm-package/assets/upload.py +++ b/npm-package/assets/upload.py @@ -3,7 +3,7 @@ Run: python3 upload.py /path/to/project Reads from .archie/: blueprint.json (required), health.json, scan.json, rules.json, - proposed_rules.json, scan_report.md (all optional). + proposed_rules.json, findings.json (all optional). Prints: shareable URL on success, warning on failure. Zero dependencies beyond Python 3.9+ stdlib. @@ -174,15 +174,11 @@ def build_bundle(project_root: Path) -> dict: if proposed: bundle["rules_proposed"] = proposed - scan_report = _read_text(archie_dir / "scan_report.md") - if scan_report: - bundle["scan_report"] = scan_report - # Structured findings from the shared accumulating store. Gives the share # viewer the 4-field shape (problem_statement/evidence/root_cause/ - # fix_direction) — far richer than the title/description regex-scraped - # from scan_report.md. Old bundles without this still fall back to the - # markdown-parsed findings. + # fix_direction). The legacy scan_report.md bundle field was retired with + # the deep-scan drift step — the viewer treats it as optional and old + # bundles that still carry it keep rendering. findings_store = _read_json(archie_dir / "findings.json") if isinstance(findings_store, dict) and isinstance(findings_store.get("findings"), list): bundle["findings"] = findings_store["findings"] @@ -195,56 +191,22 @@ def build_bundle(project_root: Path) -> dict: if isinstance(c4, dict): bundle["c4"] = c4 - # Structured semantic duplications. Two upstream sources, merged here so - # the share viewer always sees a single authoritative count regardless of - # which scan path produced them: - # - # 1. .archie/semantic_duplications.json — Agent C's output. Present - # when scan_report.md exists (written by deep-scan Phase 4). - # 2. .archie/drift_report.json deep_findings tagged - # "semantic_duplication" — /archie-deep-scan's deep-drift agent. - # This was getting silently dropped: the file existed, the findings - # were tagged, but upload.py never looked at them. Visible symptom - # was "0 semantic reimplementations" on the share cover for projects - # whose dups came from deep-drift only. + # Structured semantic duplications — legacy source only. + # .archie/semantic_duplications.json was Agent C's output from the retired + # scan flow; the deep-scan drift agent (the other historical source, via + # drift_report.json) was removed along with the drift step, and no current + # pipeline emits semantic duplications. Projects that still carry the + # legacy file keep their count on the share cover; everyone else falls back + # to health.json's mechanical line-clone metric. # # Distinct from health.json's textual duplicates (line-identical copy- - # paste). Both sources describe near-twin functions / reimplementations. - duplications: list = [] - saw_any_source = False + # paste): this field describes near-twin functions / reimplementations. sem = _read_json(archie_dir / "semantic_duplications.json") if sem and isinstance(sem.get("duplications"), list): - saw_any_source = True - duplications.extend(sem["duplications"]) - - drift = _read_json(archie_dir / "drift_report.json") - if isinstance(drift, dict): - deep_findings = drift.get("deep_findings") or [] - if isinstance(deep_findings, list): - saw_any_source = True - for f in deep_findings: - if not isinstance(f, dict): - continue - # The deep-drift agent tags entries via a `type` field - # ("semantic_duplication", "missing_pattern", etc.). Some - # older / hand-rolled outputs use a `tags` array instead; - # accept both shapes so the count is robust to schema drift. - ftype = f.get("type") - tags = f.get("tags") or [] - is_sem = ( - (isinstance(ftype, str) and "semantic_duplication" in ftype) - or (isinstance(tags, list) - and any(isinstance(t, str) and "semantic_duplication" in t for t in tags)) - ) - if is_sem: - duplications.append(f) - # Set the field whenever either source provided ANY signal — including - # the explicit "structured zero" case (empty list with both sources - # checked). Without saw_any_source, an empty result here would drop the - # field entirely and let the share viewer fall through to its prose - # heuristic, which can return non-zero from unrelated scan_report text. - if saw_any_source: - bundle["semantic_duplications"] = duplications + # Including the explicit "structured zero" (empty list): without it the + # share viewer falls through to its prose heuristic, which can return + # non-zero from unrelated report text. + bundle["semantic_duplications"] = sem["duplications"] return bundle diff --git a/npm-package/assets/workflow/deep-scan/SKILL.md b/npm-package/assets/workflow/deep-scan/SKILL.md index 3bd31f51..9551fd8c 100644 --- a/npm-package/assets/workflow/deep-scan/SKILL.md +++ b/npm-package/assets/workflow/deep-scan/SKILL.md @@ -1,6 +1,6 @@ --- name: archie-deep-scan -description: Comprehensive architecture baseline scan (15-20 min). Two-wave AI analysis producing blueprint.json, per-folder CLAUDE.md, AI-synthesized rules, health metrics, and drift detection. Use for first-time baselines or major refactors. +description: Comprehensive architecture baseline scan. Two-wave AI analysis producing blueprint.json, per-folder CLAUDE.md, AI-synthesized rules, and health metrics. Use for first-time baselines or major refactors. --- # Archie Deep Scan — Comprehensive Architecture Baseline @@ -55,7 +55,7 @@ Check the user's message (ARGUMENTS) for flags: > > **Treat every item-count anywhere in this workflow — `N-M`, "up to N", "top N", "the N most", "soft floor of N", "a handful", "the most important" — as a FLOOR with no ceiling.** Produce every item that genuinely meets its quality bar; never trim to a number, never stop at a "natural" count. The quality bar is unchanged: do NOT pad, invent, or weaken per-item rigor to inflate counts. > -> Only two limits survive comprehensive mode: **(1)** the architecture diagram stays **8-12 nodes** (readability), and **(2)** the scripts' mechanical safety/context budgets (per-file read size, per-batch token budget, recursion). Everything else — rules, findings, pitfalls, decisions, trade-offs, components, guidelines, examples, drift findings, naming examples, per-field prose length — is uncapped. +> Only two limits survive comprehensive mode: **(1)** the architecture diagram stays **8-12 nodes** (readability), and **(2)** the scripts' mechanical safety/context budgets (per-file read size, per-batch token budget, recursion). Everything else — rules, findings, pitfalls, decisions, trade-offs, components, guidelines, examples, naming examples, per-field prose length — is uncapped. > > Whenever a step dispatches a sub-agent in comprehensive depth, prepend the contract line shown at that dispatch site so the sub-agent inherits this rule. @@ -129,7 +129,7 @@ STATUS=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state | 6 | AI rule synthesis | | 7 | Intent Layer | | 8 | Cleanup | - | 9 | Drift detection | + | 9 | Finalize (health + telemetry) | Ask the user how to proceed — {{>ask_user}}: - **question:** (build dynamically) `"A previous deep-scan stopped after Step {LAST} ({step_name})."` — and if `ENRICH_DONE > 0`, append `" The Intent Layer got {ENRICH_DONE} folders in before stopping."` — then `"What do you want to do?"` @@ -191,7 +191,6 @@ If `START_STEP > N` (the Preamble decided to skip earlier steps), do not Read or | 6 | AI rule synthesis | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-6-rule-synthesis.md` | | 7 | Intent Layer — per-folder CLAUDE.md | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-7-intent-layer.md` | | 8 | Cleanup | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-8-cleanup.md` | -| 9 | Drift detection & architectural assessment | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-9-drift.md` | -| 10 | Final telemetry flush | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-10-telemetry.md` | +| 9 | Finalize — health metrics, baseline, telemetry flush | `{{WORKFLOW_ROOT}}/deep-scan/steps/step-9-finalize.md` | Step 3's `orchestration.md` in turn references four sub-agent prompt files plus a shared `grounding-rules.md` (all under `{{WORKFLOW_ROOT}}/deep-scan/steps/step-3-wave1/`) — read those as the orchestration instructs. diff --git a/npm-package/assets/workflow/deep-scan/fragments/telemetry-conventions.md b/npm-package/assets/workflow/deep-scan/fragments/telemetry-conventions.md index 58e08624..67ca3120 100644 --- a/npm-package/assets/workflow/deep-scan/fragments/telemetry-conventions.md +++ b/npm-package/assets/workflow/deep-scan/fragments/telemetry-conventions.md @@ -1,6 +1,6 @@ ## Telemetry conventions -Every Step 1–9 records its start timestamp to disk via `telemetry.py mark` as its first action. The mark auto-closes the previous step's `completed_at`, mirroring the "next step's start = prior step's end" convention. Step 9 finishes its own completion with `telemetry.py finish`. After Step 10, `telemetry.py write` consumes the persisted `.archie/telemetry/_current_run.json` and emits the final `deep-scan_.json`. +Every Step 1–9 records its start timestamp to disk via `telemetry.py mark` as its first action. The mark auto-closes the previous step's `completed_at`, mirroring the "next step's start = prior step's end" convention. Step 9 finishes its own completion with `telemetry.py finish`, then `telemetry.py write` consumes the persisted `.archie/telemetry/_current_run.json` and emits the final `deep-scan_.json`. Shell-variable fallback: the existing `TELEMETRY_STEPN_START` shell variables are still set for readability, but they are **not load-bearing** — the disk file is the source of truth. This makes the pipeline safe to `/compact` mid-run without losing timing data. diff --git a/npm-package/assets/workflow/deep-scan/steps/step-10-telemetry.md b/npm-package/assets/workflow/deep-scan/steps/step-10-telemetry.md deleted file mode 100644 index 98d2be13..00000000 --- a/npm-package/assets/workflow/deep-scan/steps/step-10-telemetry.md +++ /dev/null @@ -1,22 +0,0 @@ -## Step 10: Write telemetry - -Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. - -If the Intent Layer was skipped (INTENT_LAYER=no), mark it so explicitly: - -```bash -if [ "$INTENT_LAYER" = "no" ]; then - python3 .archie/telemetry.py extra "$PROJECT_ROOT" intent_layer skipped=true -fi -``` - -Then flush the in-flight file into the final `.archie/telemetry/deep-scan_.json`: - -```bash -python3 .archie/telemetry.py finish "$PROJECT_ROOT" -python3 .archie/telemetry.py write "$PROJECT_ROOT" -``` - -`write` auto-closes any still-open step with `now`, emits the final timestamped JSON, then deletes `_current_run.json` so the next deep-scan starts fresh. If telemetry fails for any reason, do not abort — telemetry is informational only. - -**Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. diff --git a/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md b/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md index 280a76c5..3984a591 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md @@ -71,6 +71,8 @@ Spawn the **Product** sub-agent only when `DOMAIN_LAW_COUNT` is greater than 0. - **If SCAN_MODE = "incremental":** > INCREMENTAL UPDATE. The architecture was previously analyzed — `$PROJECT_ROOT/.archie/blueprint.json` is the current full architecture and `$PROJECT_ROOT/.archie/blueprint_raw.json` carries the structural changes from Step 4. These files changed: [list `changed_files`]. Update ONLY the sections you own that are affected by these changes, and return ONLY what changed — unchanged sections are preserved by the patch merge. Use the 4-field contract (`problem_statement`, `evidence`, `root_cause`, `fix_direction`) when writing finding or pitfall entries. + The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. + **Output contract — append to each prompt, substituting that sub-agent's output path from the table as the "file path named above":** ``` diff --git a/npm-package/assets/workflow/deep-scan/steps/step-5b-risk.md b/npm-package/assets/workflow/deep-scan/steps/step-5b-risk.md index a933ec21..b696dddf 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-5b-risk.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-5b-risk.md @@ -31,6 +31,8 @@ The `f_0001` shape we are guarding against: AI sees an AGENTS.md mandate, finds **APPROACH — anchor synthesis to documented invariants.** Instead of speculatively asking *"what could go wrong?"*, walk the documented invariants in AGENTS.md, root `CLAUDE.md`, per-folder `CLAUDE.md` (Anti-Patterns and Patterns sections — `.archie/maintainer_guardrails.json` if available), and `blueprint.pitfalls`. For each invariant, ask: *"is there code in this corpus that violates it? Quote it verbatim."* If yes ⇒ that quote is the `triggering_call_site` of a finding. If the invariant is real but uniformly enforced ⇒ no finding (and no need to re-emit a pitfall already in the store). This adversarial framing converts the loose "find problems" task into a falsifiable evidence-gathering pass. +**Recency sweep (only when the mode preamble carries a "These files changed:" list — incremental runs).** That list is your sweep scope: Read each listed file (skip ones that no longer exist) plus its folder's `CLAUDE.md` and parent folder's `CLAUDE.md` when they exist. Check each file against the documented invariants exactly as above — the per-folder Patterns/Anti-Patterns are first-class invariants here, alongside `blueprint.json` decisions, trade-offs (`violation_signals`), and pitfalls (`stems_from`). A violation with a verbatim caller becomes a normal finding (full 4-field shape + `triggering_call_site`); folder-pattern erosion with no firing call site is a pitfall (upgrade the existing one if the class is already tracked). The list is bounded by definition — read every file on it; do not sample. + **Primary goal — emit NEW findings.** You have the overall picture (all Wave 1 output plus source files). Your highest-leverage work is surfacing problems that are NOT already in findings.json — things only visible from the whole-system view: cross-component coupling, pattern breakdowns that individual agents miss, constraint violations implied by the decision chain, gaps between what the blueprint claims and what the code does. Spend the bulk of your cognitive budget here. For each new finding: next-free `f_NNNN` id, `first_seen` = today, `confirmed_in_scan` = 1, `depth: "canonical"`, `source: "deep:synthesis"`, AND a non-empty `triggering_call_site`. **Novelty check before emitting.** Before you add a "new" finding, verify it is genuinely new: scan the existing store for any entry with overlapping `problem_statement` meaning OR overlapping `applies_to` files. If the same problem is already tracked under a different wording, DO NOT mint a new id — instead upgrade the existing entry (see below). A new finding must describe something the store doesn't already cover. diff --git a/npm-package/assets/workflow/deep-scan/steps/step-7-intent-layer.md b/npm-package/assets/workflow/deep-scan/steps/step-7-intent-layer.md index 67e499b9..35752b1e 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-7-intent-layer.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-7-intent-layer.md @@ -9,7 +9,7 @@ TELEMETRY_STEP7_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") **If START_STEP > 7, skip this step.** -**If `INTENT_LAYER=no` (user opted out in Step E), skip this entire step.** Print a one-line note to the user: *"Intent Layer skipped (no per-folder CLAUDE.md generated). Root CLAUDE.md + rule files still written. You can run `{{COMMAND_PREFIX}}archie-intent-layer` later if you change your mind."* Then proceed to Step 8. The `intent_layer` telemetry step will record zero elapsed time (its `started_at == completed_at`) and carry `"skipped": true` (see Step 10). +**If `INTENT_LAYER=no` (user opted out in Step E), skip this entire step.** Print a one-line note to the user: *"Intent Layer skipped (no per-folder CLAUDE.md generated). Root CLAUDE.md + rule files still written. You can run `{{COMMAND_PREFIX}}archie-intent-layer` later if you change your mind."* Then proceed to Step 8. The `intent_layer` telemetry step will record zero elapsed time (its `started_at == completed_at`) and carry `"skipped": true` (see Step 9). **If `INTENT_LAYER=yes`, execute this step fully. Do NOT ask the user whether to run, skip, or reduce scope. Do NOT offer alternatives. Run all batches as instructed below.** @@ -65,7 +65,7 @@ Then execute Phases 1–4 from that file, using `PROJECT_ROOT` in place of `$PWD ### ✓ Compact Checkpoint C — after Intent Layer -Only meaningful when `INTENT_LAYER=yes`. Step 7 has just pushed dozens-to-hundreds of {{ANALYSIS_MODEL}} subagent transcripts into conversation context; those are now fully persisted to `.archie/enrichments/*.json` and merged into per-folder `CLAUDE.md` files. Compacting here gives Step 9 (Drift Assessment) a fresh context, which matters because drift assessment reads blueprint + drift_report + CLAUDE.md files and benefits from focused attention. +Only meaningful when `INTENT_LAYER=yes`. Step 7 has just pushed dozens-to-hundreds of {{ANALYSIS_MODEL}} subagent transcripts into conversation context; those are now fully persisted to `.archie/enrichments/*.json` and merged into per-folder `CLAUDE.md` files. Compacting here gives the remaining bookkeeping steps (Cleanup, Finalize) a fresh context, so the run finishes reliably even after a very large Intent Layer pass. If `INTENT_LAYER=no` (opted out in Step E), skip this checkpoint — Checkpoint A already covered it. diff --git a/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md b/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md index 0c01a748..fe4e15b5 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md @@ -12,6 +12,12 @@ TELEMETRY_STEP8_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") rm -f .archie/tmp/archie_sub*_$PROJECT_NAME.json .archie/tmp/archie_rules_$PROJECT_NAME.json .archie/tmp/archie_intent_prompt_${PROJECT_NAME}_*.txt .archie/tmp/archie_enrichment_${PROJECT_NAME}_*.json ``` +Remove artifacts from the retired drift step (projects upgraded from older Archie versions may still carry them; nothing reads these anymore): + +```bash +rm -f .archie/drift_report.json .archie/scan_report.md +``` + ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 8 ``` diff --git a/npm-package/assets/workflow/deep-scan/steps/step-9-drift.md b/npm-package/assets/workflow/deep-scan/steps/step-9-drift.md deleted file mode 100644 index e2565969..00000000 --- a/npm-package/assets/workflow/deep-scan/steps/step-9-drift.md +++ /dev/null @@ -1,198 +0,0 @@ -## Step 9: Drift Detection & Architectural Assessment - -**Telemetry:** -```bash -python3 .archie/telemetry.py mark "$PROJECT_ROOT" deep-scan drift -TELEMETRY_STEP9_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -``` - -**If START_STEP > 9, skip this step.** - -### Phase 0: Health measurement - -```bash -python3 .archie/measure_health.py "$PROJECT_ROOT" > "$PROJECT_ROOT/.archie/health.json" 2>/dev/null -``` - -Save health scores to history for trending: - -```bash -python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep -``` - -### Phase 1: Mechanical drift scan - -```bash -python3 .archie/drift.py "$PROJECT_ROOT" -``` - -### Phase 2: Deep architectural drift (AI) - -Set the drift window by depth. Default depth: last 30 days, capped at 100 files. When `DEPTH=comprehensive`: full history, effectively unbounded. The pipeline shape stays identical in both depths — only these two vars change. Use a bash **array** for the `--since` argument so the value (which contains a space) is passed as a single argument, not word-split. - -```bash -# Re-derive DEPTH from persisted state so this late step does not depend on the -# shell variable surviving from Phase 0 across a long run / compaction. -DEPTH=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state.json --query .run_context.depth 2>/dev/null) -if [ "$DEPTH" = "comprehensive" ]; then - SINCE_ARGS=() # full history - DRIFT_MAX=1000000 # effectively unbounded -else - SINCE_ARGS=(--since="30 days ago") - DRIFT_MAX=100 -fi -``` - -Identify files to analyze: -```bash -git -C "$PROJECT_ROOT" log --name-only --pretty=format: ${SINCE_ARGS[@]+"${SINCE_ARGS[@]}"} -- '*.kt' '*.java' '*.swift' '*.ts' '*.tsx' '*.py' '*.go' '*.rs' \ - | sort -u | python3 .archie/intent_layer.py filter-ignored "$PROJECT_ROOT" | head -n "$DRIFT_MAX" -``` -If that returns nothing (new repo or no recent changes), use all source files from the scan: -```bash -python3 .archie/extract_output.py recent-files "$PROJECT_ROOT/.archie/scan.json" -``` - -For each file (batch into groups of ~15), collect: -- The file's content -- Its folder's CLAUDE.md **if it exists** (per-folder patterns, anti-patterns — these were generated in Step 7, but may be missing if Step 7 was skipped or partially completed) -- Its parent folder's CLAUDE.md **if it exists** - -Read `$PROJECT_ROOT/.archie/blueprint.json` — specifically `decisions.key_decisions`, `decisions.decision_chain`, `decisions.trade_offs` (with `violation_signals`), `pitfalls` (with `stems_from`), `communication.patterns`, `development_rules`. - -Read `$PROJECT_ROOT/.archie/drift_report.json` (mechanical findings from Phase 1). - -Spawn a **{{ANALYSIS_MODEL}} subagent** with the file contents, their folder CLAUDE.md files, and the blueprint context. {{>dispatch_single}} Tell it: - -> You are an architecture reviewer. You have the project's architectural blueprint (decisions, trade-offs, pitfalls, patterns), per-folder CLAUDE.md files describing expected patterns, mechanical drift findings (already detected), and source files to review. -> -> Find **deep architectural violations** — problems that pattern matching cannot catch. For each finding, return: -> - `folder`: the folder path -> - `file`: the specific file -> - `type`: one of `decision_violation`, `pattern_erosion`, `trade_off_undermined`, `pitfall_triggered`, `responsibility_leak`, `abstraction_bypass`, `semantic_duplication` -> - `severity`: `error` or `warn` -> - `decision_or_pattern`: which architectural decision, pattern, or pitfall this violates (reference by name from the blueprint) -> - `evidence`: the specific code (function name, class, line pattern) that demonstrates the violation -> - `message`: one sentence explaining what's wrong and why it matters -> -> Focus on: -> 1. **Decision violations** — code that contradicts a key architectural decision -> 2. **Pattern erosion** — code that doesn't follow the patterns described in its folder's CLAUDE.md -> 3. **Trade-off undermining** — code that works against an accepted trade-off (check `violation_signals`) -> 4. **Pitfall triggers** — code that falls into a documented pitfall (check `stems_from` chains) -> 5. **Responsibility leaks** — a component doing work that belongs to another component -> 6. **Abstraction bypass** — code reaching through a layer instead of using the intended interface -> 7. **Semantic duplication** — functions/methods with different signatures but essentially the same logic. AI agents frequently copy-paste a function, tweak the name/parameters, and leave the body identical or near-identical. Look for: functions with similar names (e.g., `getText`/`getTexts`, `loadUser`/`fetchUser`), functions in different files that do the same thing with slightly different types, helper functions reimplemented instead of shared. For each, use type `semantic_duplication` and explain what's duplicated and which function should be the canonical one. -> 8. **Schema drift** — when `blueprint.data_models` is non-empty, give it elevated attention. Touched files inside any `data_models[*].location` or `persistence_stores[*].migrations_dir` are signal-rich and merit extra scrutiny: did a migration ship without the corresponding ORM-model update (or vice versa)? Did a new column appear in code without a migration? Does a write path bypass the documented `lifecycle.how_to_read` repository? Use type `decision_violation` (when the change contradicts a documented invariant in `data_models[*].invariants`) or `pattern_erosion` (when it contradicts `data_models[*].lifecycle`). -> -> Do NOT report: style/formatting/naming (the script handles those), generic best-practice violations not grounded in THIS project's blueprint, or issues already in the mechanical drift report. -> -> Return JSON: `{"deep_findings": [...]}` - -Instruct the reviewer subagent to write its own output (append to its prompt). The "file path named above" is `.archie/tmp/archie_deep_drift.json`: - -``` ---- -OUTPUT CONTRACT (mandatory): -{{>output_contract}} -``` - -After the agent's confirmation returns, extract and clean up: - -```bash -python3 .archie/extract_output.py deep-drift .archie/tmp/archie_deep_drift.json "$PROJECT_ROOT/.archie/drift_report.json" -rm -f .archie/tmp/archie_deep_drift.json -``` - -### Phase 3: Present the combined assessment - -Read `$PROJECT_ROOT/.archie/blueprint.json` and `$PROJECT_ROOT/.archie/drift_report.json` (now contains both mechanical and deep findings). This is the final output — make it valuable. - -#### Part 1: What was generated - -List the generated artefacts with counts: -- Blueprint sections populated (out of total) -- Components discovered -- Enforcement rules generated -- Per-folder CLAUDE.md files created -- Rule files in `.claude/rules/` - -#### Part 2: Architecture Summary - -From the blueprint, summarize in 5-10 lines: -- **Architecture style** (from `meta.architecture_style`) -- **Key components** (top 5-7 from `components.components` — name + one-line responsibility). In comprehensive depth (`DEPTH=comprehensive`), list all (no top-N cap). -- **Technology stack highlights** (from `technology.stack` — framework, language, key libs) -- **Key decisions** (from `decisions.key_decisions` — the 2-3 most impactful, one line each). In comprehensive depth (`DEPTH=comprehensive`), list all (no top-N cap). - -#### Part 3: Architecture Health Assessment - -Rate and explain each dimension (use these exact labels: Strong / Adequate / Weak / Not assessed): - -1. **Separation of concerns** — Are layers/modules clearly bounded? Do components have single responsibilities? Any god classes or circular dependencies? -2. **Dependency direction** — Do dependencies flow in one direction? Are domain/core layers independent of infrastructure? Any inverted or tangled dependencies? -3. **Pattern consistency** — Is the same pattern used consistently across similar components? Are there one-off deviations that break the uniformity? -4. **Testability** — Is the architecture conducive to testing? Can components be tested in isolation? Are external dependencies injectable? -5. **Change impact radius** — When a component changes, how many others are affected? Are changes localised or do they ripple? - -Base every rating on actual evidence from the blueprint and drift findings — reference specific components, patterns, or findings. If the blueprint lacks data for a dimension, say "Not assessed" rather than guessing. - -#### Part 4: Architectural Drift - -Present ALL findings — mechanical and deep together, organized by severity (errors first). - -**Deep architectural findings** (from AI analysis): -- For each: the file, which decision/pattern it violates, the evidence, and why it matters -- Group related findings (e.g., multiple files violating the same decision) - -**Mechanical findings** (from script): -- Pattern divergences, dependency violations, naming violations, structural outliers, anti-pattern clusters -- For each: what diverged, why it matters, suggested action - -If 0 findings, say so — that's a positive signal. - -#### Part 5: Top Risks & Recommendations - -Synthesize from pitfalls, trade-offs, drift findings (both mechanical and deep), and your observations. List the **3-5 most important architectural risks**, ordered by impact (in comprehensive depth (`DEPTH=comprehensive`), list all — no top-N cap): -- What the risk is (one sentence) -- Where it manifests (specific components/files/drift findings) -- What to watch for going forward - -#### Part 6: Semantic Duplication - -**This is a critical section.** The mechanical verbosity score (0-1) only catches exact line-for-line clones. AI agents frequently create near-identical functions with slightly different names, signatures, or types — the verbosity metric completely misses these. - -Present the `semantic_duplication` findings from the deep drift analysis. If the drift agent found none, **do your own quick check now**: scan the skeletons for functions with similar names (e.g., `getText`/`getTexts`, `loadUser`/`fetchUser`, `formatDate` in multiple files, `handleError` reimplemented per-module). Read suspicious pairs and confirm whether the logic is duplicated. - -For each confirmed duplicate group: -- The canonical function (the one that should be the shared version) -- The duplicates: which files, what differs (just the signature? types? minor logic?) -- Whether they could be consolidated - -Present in the health table as: -``` -| Semantic duplication | N groups found | See Part 6 for details | -``` - -If genuinely none found after checking, say "No semantic duplication detected after AI analysis." - -**Health scores** from Phase 0 have been saved to `.archie/health_history.json` for trending. Note: the verbosity metric is mechanical (exact line clones only) — the semantic duplication analysis in Part 6 above is the AI-powered complement. - -### Phase 4: Persist findings to `.archie/scan_report.md` - -The Phase 3 synthesis above is valuable but ephemeral — it only exists in the chat output. `{{COMMAND_PREFIX}}archie-share` (and future trending runs of `{{COMMAND_PREFIX}}archie-deep-scan`) need the findings on disk. Write the same content to `.archie/scan_report.md` as ranked findings. - -Check whether a prior scan report exists (for resolved/new/recurring classification): -```bash -test -f "$PROJECT_ROOT/.archie/scan_report.md" && echo "PRIOR_REPORT_EXISTS" || echo "FIRST_BASELINE" -``` - -If `FIRST_BASELINE` (no prior scan_report.md): all findings are tagged **NEW (baseline)**. If `PRIOR_REPORT_EXISTS`: compare against the prior file's Findings section and classify each as **NEW**, **RECURRING**, or **RESOLVED**. - -Read `$PROJECT_ROOT/.archie/health.json` for precise numeric values and `$PROJECT_ROOT/.archie/health_history.json` to compute trends (previous run values vs. current). - -Write `$PROJECT_ROOT/.archie/scan_report.md` using the template at -`{{WORKFLOW_ROOT}}/deep-scan/templates/scan-report.md` (path relative -to the project root). Read that file first if you haven't already, then -substitute the project-specific values into the placeholders before writing -the final report. diff --git a/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md b/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md new file mode 100644 index 00000000..8037b272 --- /dev/null +++ b/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md @@ -0,0 +1,70 @@ +## Step 9: Finalize — health metrics, baseline, telemetry + +**Telemetry:** +```bash +python3 .archie/telemetry.py mark "$PROJECT_ROOT" deep-scan finalize +TELEMETRY_STEP9_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +``` + +**If START_STEP > 9, skip this step.** + +### Phase 1: Health measurement + +```bash +python3 .archie/measure_health.py "$PROJECT_ROOT" > "$PROJECT_ROOT/.archie/health.json" 2>/dev/null +``` + +Save health scores to history for trending: + +```bash +python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep +``` + +### Phase 2: Mark the run complete + +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 +``` + +Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE +``` +(Replace SCAN_MODE with the actual mode — "full" or "incremental") + +### Phase 3: Write telemetry + +Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. + +If the Intent Layer was skipped (INTENT_LAYER=no), mark it so explicitly: + +```bash +if [ "$INTENT_LAYER" = "no" ]; then + python3 .archie/telemetry.py extra "$PROJECT_ROOT" intent_layer skipped=true +fi +``` + +Then flush the in-flight file into the final `.archie/telemetry/deep-scan_.json`: + +```bash +python3 .archie/telemetry.py finish "$PROJECT_ROOT" +python3 .archie/telemetry.py write "$PROJECT_ROOT" +``` + +`write` auto-closes any still-open step with `now`, emits the final timestamped JSON, then deletes `_current_run.json` so the next deep-scan starts fresh. If telemetry fails for any reason, do not abort — telemetry is informational only. + +**Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. + +### Phase 4: Closing summary + +Present a short wrap-up to the user — a receipt, not a report (10 lines or fewer). State what the scan produced, with counts read via the allowlisted inspect commands (NEVER inline Python): + +```bash +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" blueprint.json --query '.components.components|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" rules.json --query '.rules|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" findings.json --query '.findings|length' +``` + +Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (you know this count from Step 7), and findings tracked in `.archie/findings.json`. Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. + +End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** diff --git a/npm-package/assets/workflow/deep-scan/templates/scan-report.md b/npm-package/assets/workflow/deep-scan/templates/scan-report.md deleted file mode 100644 index ec96d672..00000000 --- a/npm-package/assets/workflow/deep-scan/templates/scan-report.md +++ /dev/null @@ -1,74 +0,0 @@ -# Archie Scan Report -> Deep scan baseline | | functions / LOC analyzed | baseline run - -## Architecture Overview - -<2-3 paragraphs from Part 2: architecture style, key components, most important decisions. Prose, not bullets.> - -## Health Scores - -| Metric | Current | Previous | Trend | What it means | -|--------|--------:|---------:|------:|---------------| -| Erosion | | | | | -| Gini | | | | | -| Top-20% | | | | | -| Verbosity | | | | | -| LOC | | | | | - - - -### Complexity Trajectory - - -## Findings - -Ranked by severity, grouped by novelty. - -### NEW (first observed this scan) - - -### RECURRING (previously documented, still present) - - -### RESOLVED - - -### Data Architecture - - -## Proposed Rules - - -``` - -Sources for Findings: -- `drift_report.json` — mechanical and deep drift findings from Phase 1 and 2 -- `blueprint.json` — `pitfalls` (each causal chain becomes a finding), `decisions.trade_offs` with violated `violation_signals` (if any appear in drift_report) -- Top complexity offenders from `health.json` (only if CC ≥ 15 or a cluster — don't list every high-CC function as a finding) - -Severity mapping: -- `error` — decision violations, inverted dependencies, cycles across architectural boundaries -- `warn` — pattern erosion, god-objects, pitfalls currently manifesting, trade-offs actively undermined -- `info` — structural observations (dependency magnets, high fan-in nodes) that aren't currently broken - -Confidence: carry forward from drift findings when available; otherwise use 0.8-0.95 for findings grounded in direct code reading, lower for inferred ones. - -Verify the write: -```bash -test -s "$PROJECT_ROOT/.archie/scan_report.md" && wc -l "$PROJECT_ROOT/.archie/scan_report.md" -``` - -Expected: non-empty file with at least 30 lines. - -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 -``` - -Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE -``` -(Replace SCAN_MODE with the actual mode — "full" or "incremental") - -End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** - diff --git a/npm-package/bin/archie.mjs b/npm-package/bin/archie.mjs index e0e7419c..beec5475 100755 --- a/npm-package/bin/archie.mjs +++ b/npm-package/bin/archie.mjs @@ -356,7 +356,7 @@ if (cleanedCount > 0) { console.log(` ${DIM}cleaned ${cleanedCount} previous Archie files${RESET}`); } -for (const script of ["_common.py", "scanner.py", "refresh.py", "intent_layer.py", "renderer.py", "install_hooks.py", "merge.py", "finalize.py", "validate.py", "viewer.py", "drift.py", "c4.py", "extract_output.py", "arch_review.py", "measure_health.py", "check_rules.py", "detect_cycles.py", "upload.py", "share_setup.py", "telemetry.py", "lint_gate.py", "code_shape.py", "rule_index.py", "align_check.py", "agent_cli.py", "verify_findings.py", "apply_verdicts.py", "migrate_blueprint_rules.py", "rule_kinds.py", "backfill_kinds.py", "config.py", "telemetry_sync.py", "update_check.py", "analytics.py", "sync.py"]) { +for (const script of ["_common.py", "scanner.py", "refresh.py", "intent_layer.py", "renderer.py", "install_hooks.py", "merge.py", "finalize.py", "validate.py", "viewer.py", "c4.py", "extract_output.py", "arch_review.py", "measure_health.py", "check_rules.py", "detect_cycles.py", "upload.py", "share_setup.py", "telemetry.py", "lint_gate.py", "code_shape.py", "rule_index.py", "align_check.py", "agent_cli.py", "verify_findings.py", "apply_verdicts.py", "migrate_blueprint_rules.py", "rule_kinds.py", "backfill_kinds.py", "config.py", "telemetry_sync.py", "update_check.py", "analytics.py", "sync.py"]) { const src = join(ASSETS, script); const dest = join(archieDir, script); if (existsSync(src)) { diff --git a/tests/test_comprehensive_mode_integration.py b/tests/test_comprehensive_mode_integration.py index e1108add..97c2472d 100644 --- a/tests/test_comprehensive_mode_integration.py +++ b/tests/test_comprehensive_mode_integration.py @@ -77,25 +77,30 @@ def test_depth_persists_and_round_trips(tmp_path): assert state["run_context"]["scan_mode"] == "incremental" -def test_filter_ignored_honors_nested_and_glob(tmp_path): +def test_ignore_matcher_honors_nested_and_glob(tmp_path): + """Ancestor-dir and glob semantics for full-path callers of is_ignored + (e.g. a git-log file list, where the file may no longer exist on disk).""" + import sys + sys.path.insert(0, "archie/standalone") + from _common import IgnoreMatcher (tmp_path / ".archieignore").write_text("vendor/\n**/generated/\n") - cmd = [sys.executable, INTENT, "filter-ignored", str(tmp_path)] - stdin = "src/a.py\nvendor/deep/b.py\npkg/generated/x.py\nsrc/c.py\n" - out = subprocess.run(cmd, input=stdin, capture_output=True, text=True, check=True) - assert [l for l in out.stdout.splitlines() if l.strip()] == ["src/a.py", "src/c.py"] + m = IgnoreMatcher(tmp_path) + assert not m.is_ignored("src/a.py") + assert m.is_ignored("vendor/deep/b.py") + assert m.is_ignored("pkg/generated/x.py") + assert not m.is_ignored("src/c.py") def test_report_caps_gate_on_depth(tmp_path): - """measure_health & drift lift their report list-caps when run_context.depth + """measure_health lifts its report list-caps when run_context.depth is comprehensive (read from deep_scan_state.json, no flag needed).""" import sys, json sys.path.insert(0, "archie/standalone") from importlib import import_module (tmp_path / ".archie").mkdir() state = tmp_path / ".archie" / "deep_scan_state.json" - for mod_name in ("measure_health", "drift"): - m = import_module(mod_name) - state.write_text(json.dumps({"run_context": {"depth": "comprehensive"}})) - assert m._is_comprehensive(str(tmp_path)) is True - state.write_text(json.dumps({"run_context": {"depth": "default"}})) - assert m._is_comprehensive(str(tmp_path)) is False + m = import_module("measure_health") + state.write_text(json.dumps({"run_context": {"depth": "comprehensive"}})) + assert m._is_comprehensive(str(tmp_path)) is True + state.write_text(json.dumps({"run_context": {"depth": "default"}})) + assert m._is_comprehensive(str(tmp_path)) is False diff --git a/tests/test_comprehensive_wiring.py b/tests/test_comprehensive_wiring.py index 212f7e48..4c01049f 100644 --- a/tests/test_comprehensive_wiring.py +++ b/tests/test_comprehensive_wiring.py @@ -34,9 +34,26 @@ def test_step6_passes_comprehensive_to_renderer(): assert DEPTH_REDERIVE in t -def test_step9_rederives_depth_from_disk(): - t = _read(STEPS / "step-9-drift.md") - assert DEPTH_REDERIVE in t and "SINCE_ARGS" in t +def test_step9_finalize_carries_salvaged_bookkeeping(): + """The finalize step owns the calls salvaged from the retired drift step: + health measurement, run completion, and the incremental baseline marker.""" + t = _read(STEPS / "step-9-finalize.md") + assert "measure_health.py" in t and "--append-history" in t + assert "complete-step 9" in t + assert "save-baseline" in t + assert "telemetry.py finish" in t and "telemetry.py write" in t + + +def test_incremental_recency_sweep_is_wired(): + """The Risk agent's recency sweep only works if the orchestration expands + changed_files verbatim into the incremental preamble AND the Risk prompt + body tells the agent to read that list. Both sides, or it's dead wiring.""" + wave2 = _read(STEPS / "step-5-wave2-reasoning.md") + assert "These files changed" in wave2 + assert "expanded verbatim" in wave2 and "recency sweep" in wave2 + risk = _read(STEPS / "step-5b-risk.md") + assert "Recency sweep" in risk + assert "These files changed" in risk # the body names the preamble marker it keys on CONTRACT = "COMPREHENSIVE MODE — be exhaustive" @@ -69,7 +86,8 @@ def test_wave1_dispatch_injects_contract(): def test_npm_package_workflow_in_sync(): """Spot-check: the npm-package mirror of the wired steps matches canonical.""" for rel in ("steps/step-1-scanner.md", "steps/step-6-rule-synthesis.md", - "steps/step-5-wave2-reasoning.md", "steps/step-3-wave1/orchestration.md"): + "steps/step-5-wave2-reasoning.md", "steps/step-5b-risk.md", + "steps/step-9-finalize.md", "steps/step-3-wave1/orchestration.md"): canon = _read(WF / rel) mirror = _read(Path("npm-package/assets/workflow/deep-scan") / rel) assert canon == mirror, f"out of sync: {rel}" diff --git a/tests/test_ignore_patterns.py b/tests/test_ignore_patterns.py index 1149056f..f68b83f9 100644 --- a/tests/test_ignore_patterns.py +++ b/tests/test_ignore_patterns.py @@ -339,11 +339,11 @@ def test_gitignore_patterns_respected(self, tmp_path): assert not any("tmp_output" in p for p in paths) -def test_filter_ignored_command(tmp_path): - import subprocess, sys +def test_is_ignored_full_paths_against_archieignore(tmp_path): + """Full-path is_ignored calls honor .archieignore dir patterns even when + the files don't exist on disk (the git-log-list caller shape).""" (tmp_path / ".archieignore").write_text("vendor/\n") - cmd = [sys.executable, "archie/standalone/intent_layer.py", "filter-ignored", str(tmp_path)] - out = subprocess.run(cmd, input="src/a.py\nvendor/b.py\nsrc/c.py\n", - capture_output=True, text=True, check=True) - lines = [l for l in out.stdout.splitlines() if l.strip()] - assert lines == ["src/a.py", "src/c.py"] + m = IgnoreMatcher(tmp_path) + assert not m.is_ignored("src/a.py") + assert m.is_ignored("vendor/b.py") + assert not m.is_ignored("src/c.py") diff --git a/tests/test_telemetry_agents.py b/tests/test_telemetry_agents.py index ca57c736..94cd9d51 100644 --- a/tests/test_telemetry_agents.py +++ b/tests/test_telemetry_agents.py @@ -53,7 +53,7 @@ def test_build_summary_structure_and_total(): {"name": "scan", "seconds": 7}, {"name": "wave2_synthesis", "seconds": 61, "agents": [{"name": "design", "seconds": 30}, {"name": "risk", "seconds": 163}]}, - {"name": "drift", "seconds": 570}, + {"name": "finalize", "seconds": 570}, ] out = telemetry.build_summary(steps) assert [s["step"] for s in out["steps"]] == [1, 2, 3] # numbered in order diff --git a/tests/test_upload.py b/tests/test_upload.py index 2dbc39b4..38544ec8 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -63,15 +63,16 @@ def test_build_bundle_includes_all(mock_archie_dir): archie.joinpath("proposed_rules.json").write_text(json.dumps({ "rules": [{"id": "p1", "description": "Use barrel exports", "confidence": 0.7}] })) - archie.joinpath("scan_report.md").write_text("# Scan\n\n## Findings\n- thing one") + # scan_report.md is retired (drift step removed) — a stale file on disk + # must NOT be bundled: stale prose would ship to the share viewer. + archie.joinpath("scan_report.md").write_text("# Scan\n\n## Findings\n- stale thing") bundle = build_bundle(mock_archie_dir) assert "blueprint" in bundle assert "health" in bundle assert "scan_meta" in bundle assert "rules_adopted" in bundle assert "rules_proposed" in bundle - assert "scan_report" in bundle - assert "## Findings" in bundle["scan_report"] + assert "scan_report" not in bundle assert bundle["blueprint"]["meta"]["repository"] == "test/repo" assert len(bundle["rules_adopted"]["rules"]) == 1 assert len(bundle["rules_proposed"]["rules"]) == 1 From aaa1e83bf35cfab96ee76589df1c5958e6bef49d Mon Sep 17 00:00:00 2001 From: Gabor Bakos Date: Fri, 12 Jun 2026 12:13:31 +0200 Subject: [PATCH 2/2] fix(deep-scan): address review findings on the drift-step removal Review (Claude adversarial + testing/maintainability/security specialists + Codex cross-model) confirmed two regressions and a set of stragglers: - Step 8's new `rm -f .archie/drift_report.json .archie/scan_report.md` was not in the Claude permission allowlist (only the tmp/ and health.json rm globs are) and would prompt mid-pipeline on every upgraded project; it was also cwd-relative and skipped per-package workspaces. Moved the retirement sweep to install.py::_clean_legacy_layout (allowlisted, PROJECT_ROOT-anchored, per-workspace) and expanded it: drift_report.json, drift_diff.json, scan_report.md, semantic_duplications.json, drift_history/. - The recency sweep died on any resumed incremental run: changed_files lives only in conversation memory and neither save-run-context nor the Resume Prelude restores it. Both the Wave 2 dispatch and the Resume Prelude now regenerate it via detect-changes (safe: save-baseline only moves at Step 9). Wiring test added. - detect-changes now filters the changed list through IgnoreMatcher - git lists tracked-but-.archieignore'd files, which inflated the incremental threshold and would have fed vendored files to the recency sweep (the deleted filter-ignored subcommand used to provide exactly this). Test added. - step-9-finalize: telemetry flush now runs BEFORE complete-step 9 so an interruption leaves status=in_progress and --continue re-runs the idempotent step instead of stranding the telemetry; the receipt reads the per-folder count from enrich_state.json (not from pre-compact memory) and includes a health line. - --from N is now bounded to 1-9 (a stale --from 10 validated clean and no-opped every step). - Dead code: upload.py _read_text removed (last caller was the scan_report read). Stale references fixed: 'Steps 1-10' in SKILL.md and scope_resolution.md, drift.py/deep-drift/recent-files/templates mentions in ARCHITECTURE.md, drift in CLAUDE.md repo layout, scan_report.md in gitignore.default, 'Top risks' block in the README sample output. - New tests: semantic_duplications bundle behavior (legacy file, structured zero, stale drift_report ignored), installer retirement sweep, a pytest gate running scripts/verify_sync.py, resume-regeneration wiring, and the detect-changes ignore filter. Suite: 985 passed. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 2 +- README.md | 3 - archie/assets/gitignore.default | 4 +- .../workflow/_shared/scope_resolution.md | 2 +- archie/assets/workflow/deep-scan/SKILL.md | 11 +-- .../deep-scan/fragments/resume-prelude.md | 5 ++ .../deep-scan/steps/step-5-wave2-reasoning.md | 6 +- .../deep-scan/steps/step-8-cleanup.md | 6 +- .../deep-scan/steps/step-9-finalize.md | 34 +++++---- archie/install.py | 20 ++++-- archie/standalone/intent_layer.py | 13 ++++ archie/standalone/upload.py | 17 ++--- docs/ARCHITECTURE.md | 6 +- npm-package/assets/_install_pkg/install.py | 20 ++++-- npm-package/assets/gitignore.default | 4 +- npm-package/assets/intent_layer.py | 13 ++++ npm-package/assets/upload.py | 17 ++--- .../workflow/_shared/scope_resolution.md | 2 +- .../assets/workflow/deep-scan/SKILL.md | 11 +-- .../deep-scan/fragments/resume-prelude.md | 5 ++ .../deep-scan/steps/step-5-wave2-reasoning.md | 6 +- .../deep-scan/steps/step-8-cleanup.md | 6 +- .../deep-scan/steps/step-9-finalize.md | 34 +++++---- tests/test_asset_sync.py | 25 +++++++ tests/test_comprehensive_wiring.py | 16 ++++- tests/test_detect_changes_ignore.py | 69 +++++++++++++++++++ tests/test_install_loop.py | 22 +++++- tests/test_upload.py | 38 ++++++++++ 28 files changed, 316 insertions(+), 101 deletions(-) create mode 100644 tests/test_asset_sync.py create mode 100644 tests/test_detect_changes_ignore.py diff --git a/CLAUDE.md b/CLAUDE.md index 1dd7e524..7f9e8f91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Archie — AI-powered architecture analysis and enforcement for coding agents. A ## Repository Layout - `archie/` — Python package (`archie-cli`): CLI commands, analysis engine, standalone scripts -- `archie/standalone/` — Zero-dependency Python scripts (scanner, renderer, validator, intent layer, health, drift, hooks) +- `archie/standalone/` — Zero-dependency Python scripts (scanner, renderer, validator, intent layer, health, hooks) - `npm-package/` — NPM distribution (`npx @bitraptors/archie`): copies scripts + Claude Code commands to target projects - `tests/` — Test suite (pytest) - `docs/` — Architecture documentation diff --git a/README.md b/README.md index 5f449753..32e0881e 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,6 @@ Run `/archie-deep-scan` once for the baseline, then `/archie-deep-scan --increme 14 warnings (sync I/O in async paths, monolith routers, compat-hook CRUD leak, …) — each verified against its triggering call site before it ships - Top risks: IPC security hole · API key exposure · stale WebSocket state · - circular deps & layer violations · duplicate startup handler - Archie is now active. Rules will be enforced on every code change. Run /archie-deep-scan --incremental after code changes to refresh the analysis. ``` diff --git a/archie/assets/gitignore.default b/archie/assets/gitignore.default index fb30f1c8..6ef6b37d 100644 --- a/archie/assets/gitignore.default +++ b/archie/assets/gitignore.default @@ -7,8 +7,8 @@ # were yours. Re-running the installer recreates them. # # The architecture snapshot you MAY want to commit lives alongside these and is -# deliberately NOT ignored: blueprint.json, rules.json, scan_report.md, -# findings.json, health.json, c4.json, etc. +# deliberately NOT ignored: blueprint.json, rules.json, findings.json, +# health.json, c4.json, etc. _install_pkg/ viewer/ diff --git a/archie/assets/workflow/_shared/scope_resolution.md b/archie/assets/workflow/_shared/scope_resolution.md index f2d3eb0c..2b8f8980 100644 --- a/archie/assets/workflow/_shared/scope_resolution.md +++ b/archie/assets/workflow/_shared/scope_resolution.md @@ -120,7 +120,7 @@ Map the answer: Yes → `INTENT_LAYER=yes`, No → `INTENT_LAYER=no`. Expose `IN ### Step F: Persist run context -Write every shell variable that Steps 1–10 depend on into `.archie/deep_scan_state.json` so `/compact` + `--continue` can rehydrate them from disk without relying on orchestrator memory: +Write every shell variable that Steps 1–9 depend on into `.archie/deep_scan_state.json` so `/compact` + `--continue` can rehydrate them from disk without relying on orchestrator memory: ```bash echo "$WORKSPACES" | python3 .archie/intent_layer.py deep-scan-state "$PWD" save-run-context \ diff --git a/archie/assets/workflow/deep-scan/SKILL.md b/archie/assets/workflow/deep-scan/SKILL.md index 9551fd8c..d173afcc 100644 --- a/archie/assets/workflow/deep-scan/SKILL.md +++ b/archie/assets/workflow/deep-scan/SKILL.md @@ -60,13 +60,14 @@ Check the user's message (ARGUMENTS) for flags: > Whenever a step dispatches a sub-agent in comprehensive depth, prepend the contract line shown at that dispatch site so the sub-agent inherits this rule. **If `--from N` is present** (e.g., `{{COMMAND_PREFIX}}archie-deep-scan --from 5`): -1. Set `START_STEP = N` (the number after --from) and `RESUME_ACTION=resume` (so the Resume Prelude rehydrates shell variables from `deep_scan_state.run_context`). -2. Validate prerequisites exist: +1. If N is not between 1 and 9, tell the user the valid range is 1-9 (the pipeline has 9 steps; older versions had a step 10, which no longer exists) and stop. +2. Set `START_STEP = N` (the number after --from) and `RESUME_ACTION=resume` (so the Resume Prelude rehydrates shell variables from `deep_scan_state.run_context`). +3. Validate prerequisites exist: ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" check-prereqs N ``` -3. If check fails, tell the user which files are missing and which earlier step to run. -4. If check passes, proceed. Do NOT call `deep-scan-state init` — it would wipe the state the Resume Prelude needs to read. +4. If check fails, tell the user which files are missing and which earlier step to run. +5. If check passes, proceed. Do NOT call `deep-scan-state init` — it would wipe the state the Resume Prelude needs to read. **If `--continue` is present:** 1. Read state (no prompt — `--continue` is an explicit opt-in): @@ -158,7 +159,7 @@ STATUS=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state **For every step below:** - If the step number < START_STEP, skip it entirely. - If SCAN_MODE is not set, it defaults to "full" (all existing behavior unchanged). -- **Do NOT ask the user any questions during Steps 1–10. Do NOT offer to skip, reduce scope, or present alternatives for any step. Execute every step fully as documented.** This rule applies ONLY to Steps 1–10. It does NOT apply to Phase 0 / Activation: the scope prompt (Step C) and Intent Layer prompt (Step E) in `scope_resolution.md` are mandatory decision gates and MUST still be asked — see below. +- **Do NOT ask the user any questions during Steps 1–9. Do NOT offer to skip, reduce scope, or present alternatives for any step. Execute every step fully as documented.** This rule applies ONLY to Steps 1–9. It does NOT apply to Phase 0 / Activation: the scope prompt (Step C) and Intent Layer prompt (Step E) in `scope_resolution.md` are mandatory decision gates and MUST still be asked — see below. ## Activation — read these before running any step diff --git a/archie/assets/workflow/deep-scan/fragments/resume-prelude.md b/archie/assets/workflow/deep-scan/fragments/resume-prelude.md index 5be14f73..ad4118cf 100644 --- a/archie/assets/workflow/deep-scan/fragments/resume-prelude.md +++ b/archie/assets/workflow/deep-scan/fragments/resume-prelude.md @@ -69,4 +69,9 @@ fi - The consistency check is defensive. Under normal compact-and-resume flow, telemetry step count ≥ last_completed always holds because each step marks its start *before* calling `complete-step N`. A warning here signals something outside the happy path (manual state edit, aborted step, corrupted file). - `WORKSPACES` is rehydrated as a newline-separated string, matching what Step C's scope picker originally produced. Downstream iteration patterns (`while IFS= read`; `printf '%s\n' "$WORKSPACES" | ...`) work identically. - If `--from N` is supplied, the orchestrator sets `FROM_STEP=N` before this block runs. +- **Incremental runs:** `changed_files`/`affected_folders` are intentionally NOT persisted (they can be large and go stale). When `SCAN_MODE=incremental`, regenerate them on resume — the baseline marker only moves at Step 9's `save-baseline`, so this reproduces the Preamble's exact output. Steps 3, 5 (the Risk agent's recency sweep depends on it), and 7 all consume this list: + + ```bash + python3 .archie/intent_layer.py deep-scan-state "$PWD" detect-changes + ``` diff --git a/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md b/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md index 3984a591..c99d97c1 100644 --- a/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md +++ b/archie/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md @@ -71,7 +71,11 @@ Spawn the **Product** sub-agent only when `DOMAIN_LAW_COUNT` is greater than 0. - **If SCAN_MODE = "incremental":** > INCREMENTAL UPDATE. The architecture was previously analyzed — `$PROJECT_ROOT/.archie/blueprint.json` is the current full architecture and `$PROJECT_ROOT/.archie/blueprint_raw.json` carries the structural changes from Step 4. These files changed: [list `changed_files`]. Update ONLY the sections you own that are affected by these changes, and return ONLY what changed — unchanged sections are preserved by the patch merge. Use the 4-field contract (`problem_statement`, `evidence`, `root_cause`, `fix_direction`) when writing finding or pitfall entries. - The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. + The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. **If the list is no longer in context** (resume after `/compact`, `--continue`, or `--from 5`), regenerate it before dispatching — the baseline marker is untouched until Step 9's `save-baseline`, so this reproduces the Preamble's exact output: + + ```bash + python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" detect-changes + ``` **Output contract — append to each prompt, substituting that sub-agent's output path from the table as the "file path named above":** diff --git a/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md b/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md index fe4e15b5..b62a609d 100644 --- a/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md +++ b/archie/assets/workflow/deep-scan/steps/step-8-cleanup.md @@ -12,11 +12,7 @@ TELEMETRY_STEP8_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") rm -f .archie/tmp/archie_sub*_$PROJECT_NAME.json .archie/tmp/archie_rules_$PROJECT_NAME.json .archie/tmp/archie_intent_prompt_${PROJECT_NAME}_*.txt .archie/tmp/archie_enrichment_${PROJECT_NAME}_*.json ``` -Remove artifacts from the retired drift step (projects upgraded from older Archie versions may still carry them; nothing reads these anymore): - -```bash -rm -f .archie/drift_report.json .archie/scan_report.md -``` +(Artifacts from the retired drift step — `drift_report.json`, `scan_report.md`, `drift_history/` — are swept by the installer's legacy cleanup at upgrade time, not here: that path is allowlisted, `$PROJECT_ROOT`-anchored, and runs for every workspace.) ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 8 diff --git a/archie/assets/workflow/deep-scan/steps/step-9-finalize.md b/archie/assets/workflow/deep-scan/steps/step-9-finalize.md index 8037b272..2966b05e 100644 --- a/archie/assets/workflow/deep-scan/steps/step-9-finalize.md +++ b/archie/assets/workflow/deep-scan/steps/step-9-finalize.md @@ -1,4 +1,4 @@ -## Step 9: Finalize — health metrics, baseline, telemetry +## Step 9: Finalize — health metrics, telemetry, baseline **Telemetry:** ```bash @@ -20,19 +20,7 @@ Save health scores to history for trending: python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep ``` -### Phase 2: Mark the run complete - -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 -``` - -Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE -``` -(Replace SCAN_MODE with the actual mode — "full" or "incremental") - -### Phase 3: Write telemetry +### Phase 2: Write telemetry Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. @@ -55,6 +43,20 @@ python3 .archie/telemetry.py write "$PROJECT_ROOT" **Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. +### Phase 3: Mark the run complete + +Deliberately AFTER the telemetry flush: `complete-step 9` flips the run's `status` to `completed`, which tells `--continue` there is nothing left to resume. If the run is interrupted before this point, status stays `in_progress` and `--continue` simply re-runs Step 9 — every command in this step is idempotent. + +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 +``` + +Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE +``` +(Replace SCAN_MODE with the actual mode — "full" or "incremental") + ### Phase 4: Closing summary Present a short wrap-up to the user — a receipt, not a report (10 lines or fewer). State what the scan produced, with counts read via the allowlisted inspect commands (NEVER inline Python): @@ -63,8 +65,10 @@ Present a short wrap-up to the user — a receipt, not a report (10 lines or few python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" blueprint.json --query '.components.components|length' python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" rules.json --query '.rules|length' python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" findings.json --query '.findings|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" enrich_state.json --query '.done|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" health.json --query .erosion ``` -Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (you know this count from Step 7), and findings tracked in `.archie/findings.json`. Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. +Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (the `enrich_state.json` query above — do not rely on remembering Step 7, a compact may have intervened), findings tracked in `.archie/findings.json`, and one health line (erosion score from `health.json`). Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** diff --git a/archie/install.py b/archie/install.py index b566dd4c..e07f217b 100644 --- a/archie/install.py +++ b/archie/install.py @@ -97,15 +97,25 @@ def _clean_legacy_layout(project_root: Path) -> None: except OSError: pass - # Retired pipeline scripts — remove on upgrade so stale copies don't linger - # in .archie/. (The npm installer wipes all .py files before copying; this - # pip path copies over without sweeping, so retired names need an explicit - # delete.) - for retired in ("drift.py",): + # Retired pipeline scripts AND drift-step data artifacts — remove on upgrade + # so stale copies don't linger in .archie/ and stale reports can't be + # mistaken for (or shipped as) current output. The npm installer wipes all + # .py files before copying, but data artifacts survive both install paths + # without this explicit sweep. This is the canonical retirement point: it + # is $PROJECT_ROOT-anchored, covered by the install allowlist, and runs for + # every workspace — the deep-scan workflow deliberately does NOT rm these. + for retired in ( + "drift.py", # mechanical drift scanner (step retired in 2.10) + "drift_report.json", # mechanical + deep drift findings + "drift_diff.json", # drift.py's snapshot diff + "scan_report.md", # LLM-written scan report (Step 9 Phase 4) + "semantic_duplications.json", # Agent C duplications; nothing emits it anymore + ): try: (project_root / ".archie" / retired).unlink() except OSError: pass + shutil.rmtree(project_root / ".archie" / "drift_history", ignore_errors=True) current_command_names = {c.name for c in COMMANDS} diff --git a/archie/standalone/intent_layer.py b/archie/standalone/intent_layer.py index b45c84e6..2868ec96 100644 --- a/archie/standalone/intent_layer.py +++ b/archie/standalone/intent_layer.py @@ -940,6 +940,19 @@ def _load_state() -> dict: except Exception: print(json.dumps({"mode": "full", "reason": "git diff failed"})) return + # Honor .archieignore/.gitignore. git diff lists tracked files even + # when they're .archieignore'd (git doesn't know that file), so + # vendored/generated paths would otherwise count toward the + # incremental threshold AND get read by the Risk agent's recency + # sweep. Best-effort: a broken ignore file must not kill detection. + try: + matcher = IgnoreMatcher(root) + changed = [ + f for f in changed + if not matcher.is_ignored(f.replace(os.sep, "/")) + ] + except Exception: + pass # Count total files scan = _load_json(root / ".archie" / "scan.json") total = len(scan.get("file_tree", [])) diff --git a/archie/standalone/upload.py b/archie/standalone/upload.py index 1a052efa..8aa9d14e 100644 --- a/archie/standalone/upload.py +++ b/archie/standalone/upload.py @@ -57,15 +57,6 @@ def _read_json(path: Path) -> dict | None: return None -def _read_text(path: Path) -> str | None: - if not path.exists(): - return None - try: - return path.read_text() - except OSError: - return None - - TOP_N_HIGH_CC = 20 TOP_N_DUPLICATES = 10 @@ -195,9 +186,11 @@ def build_bundle(project_root: Path) -> dict: # .archie/semantic_duplications.json was Agent C's output from the retired # scan flow; the deep-scan drift agent (the other historical source, via # drift_report.json) was removed along with the drift step, and no current - # pipeline emits semantic duplications. Projects that still carry the - # legacy file keep their count on the share cover; everyone else falls back - # to health.json's mechanical line-clone metric. + # pipeline emits semantic duplications. The installer's legacy sweep + # (install.py::_clean_legacy_layout) deletes the file on upgrade, so this + # read only fires for shares made before the next `npx @bitraptors/archie` + # run; after that the share cover falls back to health.json's mechanical + # line-clone metric. Kept so pre-upgrade shares don't lose data mid-flight. # # Distinct from health.json's textual duplicates (line-identical copy- # paste): this field describes near-twin functions / reimplementations. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 49ec1456..48be283c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -358,7 +358,7 @@ Project-specific bulks (e.g. a 20k-key product catalog, a giant CLDR dump) are a Each command is a thin shim (`.claude/commands/archie-*.md` for Claude, `.agents/skills/archie-*/SKILL.md` for Codex) that points at the command's rendered workflow under `.archie/workflow///SKILL.md`. `/archie-deep-scan`'s `SKILL.md` is an orchestrator/router with self-contained per-step files (`steps/`, `fragments/`, `templates/`); the other commands have a single `SKILL.md`. The step split keeps each step independently readable and lets the pipeline survive `/compact` and resume via `--continue` / `--from N`. The orchestrator pattern: -1. Calling standalone Python scripts for deterministic steps (`scanner.py`, `measure_health.py`, `detect_cycles.py`, `finalize.py`, `extract_output.py`, `drift.py`, `intent_layer.py`, `telemetry.py`). +1. Calling standalone Python scripts for deterministic steps (`scanner.py`, `measure_health.py`, `detect_cycles.py`, `finalize.py`, `extract_output.py`, `intent_layer.py`, `telemetry.py`). 2. Spawning subagents via the Agent tool for AI steps (3–4 parallel Sonnets in Wave 1, one Opus in Wave 2, one Sonnet for rule synthesis, N Sonnets for Intent Layer if opted in). 3. Using `AskUserQuestion` for all single-choice prompts (scope picker, parallel/sequential, Intent Layer opt-in) — no free-text answers to parse. @@ -687,7 +687,7 @@ Zero-dependency Python scripts in `archie/standalone/`. These are exported to ta | `migrate_blueprint_rules.py` | Migrates legacy blueprint-derived rule sections (pre-3.0) into `proposed_rules.json` | | `arch_review.py` | Architectural review checklist for plans and diffs | | `refresh.py` | File change detection (hash comparison) | -| `extract_output.py` | Subcommands: `rules`, `deep-drift`, `recent-files`, `save-duplications` | +| `extract_output.py` | Subcommands: `rules`, `save-duplications` | | `telemetry.py` | Per-run step-level wall-clock timing → `.archie/telemetry/_.json`. Subcommands: `mark`, `finish`, `extra`, `read`, `write`, `clear`, `steps-count` | | `telemetry_sync.py` | Anonymous opt-in usage telemetry — records events to `~/.archie/analytics/runs.jsonl` and pushes to the Supabase `telemetry-ingest` function. Subcommands incl. `record-event`, `record-install`, `post-run`, `status`, `purge` | | `update_check.py` | Anonymous opt-in npm-registry update check (cached, snooze ladder 24h→48h→7d). Prints `UPGRADE_AVAILABLE` / `JUST_UPGRADED` markers for slash-command preambles | @@ -900,7 +900,7 @@ Slash commands at `.claude/commands/` — 4 shim files written by `ClaudeConnect All choice prompts in scan/deep-scan use the `{{>ask_user}}` partial, which renders to `AskUserQuestion` for Claude (monorepo scope picker, parallel/sequential, Intent Layer opt-in) — no free-text answers to parse. The scope (Step C) and Intent Layer (Step E) prompts are **mandatory decision gates**: the workflow instructs the agent to ask them even under a "no clarifying questions" harness mode, since they change which trees get scanned and what files get written. -The rendered deep-scan tree (`.archie/workflow/claude/deep-scan/`) is the pipeline itself: `SKILL.md` is the orchestrator/router; `steps/` holds one file per pipeline step (Step 3 fans out into `step-3-wave1/` with a prompt file per Wave 1 agent); `fragments/` holds the cross-step telemetry and `/compact`-resume contracts; `templates/` holds the scan-report template. The dispatch partials render to Agent tool calls — Wave-1 fans out by emitting all Agent tool calls in a single message. +The rendered deep-scan tree (`.archie/workflow/claude/deep-scan/`) is the pipeline itself: `SKILL.md` is the orchestrator/router; `steps/` holds one file per pipeline step (Step 3 fans out into `step-3-wave1/` with a prompt file per Wave 1 agent); `fragments/` holds the cross-step telemetry and `/compact`-resume contracts. The dispatch partials render to Agent tool calls — Wave-1 fans out by emitting all Agent tool calls in a single message. `ClaudeConnector.finalize` writes hook event registrations + the `permissions.allow` array into `.claude/settings.local.json` so `python3 .archie/*.py`, `Agent(*)`, `Read(.archie/**)`, `Write(**/CLAUDE.md)` and the rest of the workflow runs prompt-free. diff --git a/npm-package/assets/_install_pkg/install.py b/npm-package/assets/_install_pkg/install.py index b566dd4c..e07f217b 100644 --- a/npm-package/assets/_install_pkg/install.py +++ b/npm-package/assets/_install_pkg/install.py @@ -97,15 +97,25 @@ def _clean_legacy_layout(project_root: Path) -> None: except OSError: pass - # Retired pipeline scripts — remove on upgrade so stale copies don't linger - # in .archie/. (The npm installer wipes all .py files before copying; this - # pip path copies over without sweeping, so retired names need an explicit - # delete.) - for retired in ("drift.py",): + # Retired pipeline scripts AND drift-step data artifacts — remove on upgrade + # so stale copies don't linger in .archie/ and stale reports can't be + # mistaken for (or shipped as) current output. The npm installer wipes all + # .py files before copying, but data artifacts survive both install paths + # without this explicit sweep. This is the canonical retirement point: it + # is $PROJECT_ROOT-anchored, covered by the install allowlist, and runs for + # every workspace — the deep-scan workflow deliberately does NOT rm these. + for retired in ( + "drift.py", # mechanical drift scanner (step retired in 2.10) + "drift_report.json", # mechanical + deep drift findings + "drift_diff.json", # drift.py's snapshot diff + "scan_report.md", # LLM-written scan report (Step 9 Phase 4) + "semantic_duplications.json", # Agent C duplications; nothing emits it anymore + ): try: (project_root / ".archie" / retired).unlink() except OSError: pass + shutil.rmtree(project_root / ".archie" / "drift_history", ignore_errors=True) current_command_names = {c.name for c in COMMANDS} diff --git a/npm-package/assets/gitignore.default b/npm-package/assets/gitignore.default index fb30f1c8..6ef6b37d 100644 --- a/npm-package/assets/gitignore.default +++ b/npm-package/assets/gitignore.default @@ -7,8 +7,8 @@ # were yours. Re-running the installer recreates them. # # The architecture snapshot you MAY want to commit lives alongside these and is -# deliberately NOT ignored: blueprint.json, rules.json, scan_report.md, -# findings.json, health.json, c4.json, etc. +# deliberately NOT ignored: blueprint.json, rules.json, findings.json, +# health.json, c4.json, etc. _install_pkg/ viewer/ diff --git a/npm-package/assets/intent_layer.py b/npm-package/assets/intent_layer.py index b45c84e6..2868ec96 100644 --- a/npm-package/assets/intent_layer.py +++ b/npm-package/assets/intent_layer.py @@ -940,6 +940,19 @@ def _load_state() -> dict: except Exception: print(json.dumps({"mode": "full", "reason": "git diff failed"})) return + # Honor .archieignore/.gitignore. git diff lists tracked files even + # when they're .archieignore'd (git doesn't know that file), so + # vendored/generated paths would otherwise count toward the + # incremental threshold AND get read by the Risk agent's recency + # sweep. Best-effort: a broken ignore file must not kill detection. + try: + matcher = IgnoreMatcher(root) + changed = [ + f for f in changed + if not matcher.is_ignored(f.replace(os.sep, "/")) + ] + except Exception: + pass # Count total files scan = _load_json(root / ".archie" / "scan.json") total = len(scan.get("file_tree", [])) diff --git a/npm-package/assets/upload.py b/npm-package/assets/upload.py index 1a052efa..8aa9d14e 100644 --- a/npm-package/assets/upload.py +++ b/npm-package/assets/upload.py @@ -57,15 +57,6 @@ def _read_json(path: Path) -> dict | None: return None -def _read_text(path: Path) -> str | None: - if not path.exists(): - return None - try: - return path.read_text() - except OSError: - return None - - TOP_N_HIGH_CC = 20 TOP_N_DUPLICATES = 10 @@ -195,9 +186,11 @@ def build_bundle(project_root: Path) -> dict: # .archie/semantic_duplications.json was Agent C's output from the retired # scan flow; the deep-scan drift agent (the other historical source, via # drift_report.json) was removed along with the drift step, and no current - # pipeline emits semantic duplications. Projects that still carry the - # legacy file keep their count on the share cover; everyone else falls back - # to health.json's mechanical line-clone metric. + # pipeline emits semantic duplications. The installer's legacy sweep + # (install.py::_clean_legacy_layout) deletes the file on upgrade, so this + # read only fires for shares made before the next `npx @bitraptors/archie` + # run; after that the share cover falls back to health.json's mechanical + # line-clone metric. Kept so pre-upgrade shares don't lose data mid-flight. # # Distinct from health.json's textual duplicates (line-identical copy- # paste): this field describes near-twin functions / reimplementations. diff --git a/npm-package/assets/workflow/_shared/scope_resolution.md b/npm-package/assets/workflow/_shared/scope_resolution.md index f2d3eb0c..2b8f8980 100644 --- a/npm-package/assets/workflow/_shared/scope_resolution.md +++ b/npm-package/assets/workflow/_shared/scope_resolution.md @@ -120,7 +120,7 @@ Map the answer: Yes → `INTENT_LAYER=yes`, No → `INTENT_LAYER=no`. Expose `IN ### Step F: Persist run context -Write every shell variable that Steps 1–10 depend on into `.archie/deep_scan_state.json` so `/compact` + `--continue` can rehydrate them from disk without relying on orchestrator memory: +Write every shell variable that Steps 1–9 depend on into `.archie/deep_scan_state.json` so `/compact` + `--continue` can rehydrate them from disk without relying on orchestrator memory: ```bash echo "$WORKSPACES" | python3 .archie/intent_layer.py deep-scan-state "$PWD" save-run-context \ diff --git a/npm-package/assets/workflow/deep-scan/SKILL.md b/npm-package/assets/workflow/deep-scan/SKILL.md index 9551fd8c..d173afcc 100644 --- a/npm-package/assets/workflow/deep-scan/SKILL.md +++ b/npm-package/assets/workflow/deep-scan/SKILL.md @@ -60,13 +60,14 @@ Check the user's message (ARGUMENTS) for flags: > Whenever a step dispatches a sub-agent in comprehensive depth, prepend the contract line shown at that dispatch site so the sub-agent inherits this rule. **If `--from N` is present** (e.g., `{{COMMAND_PREFIX}}archie-deep-scan --from 5`): -1. Set `START_STEP = N` (the number after --from) and `RESUME_ACTION=resume` (so the Resume Prelude rehydrates shell variables from `deep_scan_state.run_context`). -2. Validate prerequisites exist: +1. If N is not between 1 and 9, tell the user the valid range is 1-9 (the pipeline has 9 steps; older versions had a step 10, which no longer exists) and stop. +2. Set `START_STEP = N` (the number after --from) and `RESUME_ACTION=resume` (so the Resume Prelude rehydrates shell variables from `deep_scan_state.run_context`). +3. Validate prerequisites exist: ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" check-prereqs N ``` -3. If check fails, tell the user which files are missing and which earlier step to run. -4. If check passes, proceed. Do NOT call `deep-scan-state init` — it would wipe the state the Resume Prelude needs to read. +4. If check fails, tell the user which files are missing and which earlier step to run. +5. If check passes, proceed. Do NOT call `deep-scan-state init` — it would wipe the state the Resume Prelude needs to read. **If `--continue` is present:** 1. Read state (no prompt — `--continue` is an explicit opt-in): @@ -158,7 +159,7 @@ STATUS=$(python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" deep_scan_state **For every step below:** - If the step number < START_STEP, skip it entirely. - If SCAN_MODE is not set, it defaults to "full" (all existing behavior unchanged). -- **Do NOT ask the user any questions during Steps 1–10. Do NOT offer to skip, reduce scope, or present alternatives for any step. Execute every step fully as documented.** This rule applies ONLY to Steps 1–10. It does NOT apply to Phase 0 / Activation: the scope prompt (Step C) and Intent Layer prompt (Step E) in `scope_resolution.md` are mandatory decision gates and MUST still be asked — see below. +- **Do NOT ask the user any questions during Steps 1–9. Do NOT offer to skip, reduce scope, or present alternatives for any step. Execute every step fully as documented.** This rule applies ONLY to Steps 1–9. It does NOT apply to Phase 0 / Activation: the scope prompt (Step C) and Intent Layer prompt (Step E) in `scope_resolution.md` are mandatory decision gates and MUST still be asked — see below. ## Activation — read these before running any step diff --git a/npm-package/assets/workflow/deep-scan/fragments/resume-prelude.md b/npm-package/assets/workflow/deep-scan/fragments/resume-prelude.md index 5be14f73..ad4118cf 100644 --- a/npm-package/assets/workflow/deep-scan/fragments/resume-prelude.md +++ b/npm-package/assets/workflow/deep-scan/fragments/resume-prelude.md @@ -69,4 +69,9 @@ fi - The consistency check is defensive. Under normal compact-and-resume flow, telemetry step count ≥ last_completed always holds because each step marks its start *before* calling `complete-step N`. A warning here signals something outside the happy path (manual state edit, aborted step, corrupted file). - `WORKSPACES` is rehydrated as a newline-separated string, matching what Step C's scope picker originally produced. Downstream iteration patterns (`while IFS= read`; `printf '%s\n' "$WORKSPACES" | ...`) work identically. - If `--from N` is supplied, the orchestrator sets `FROM_STEP=N` before this block runs. +- **Incremental runs:** `changed_files`/`affected_folders` are intentionally NOT persisted (they can be large and go stale). When `SCAN_MODE=incremental`, regenerate them on resume — the baseline marker only moves at Step 9's `save-baseline`, so this reproduces the Preamble's exact output. Steps 3, 5 (the Risk agent's recency sweep depends on it), and 7 all consume this list: + + ```bash + python3 .archie/intent_layer.py deep-scan-state "$PWD" detect-changes + ``` diff --git a/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md b/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md index 3984a591..c99d97c1 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-5-wave2-reasoning.md @@ -71,7 +71,11 @@ Spawn the **Product** sub-agent only when `DOMAIN_LAW_COUNT` is greater than 0. - **If SCAN_MODE = "incremental":** > INCREMENTAL UPDATE. The architecture was previously analyzed — `$PROJECT_ROOT/.archie/blueprint.json` is the current full architecture and `$PROJECT_ROOT/.archie/blueprint_raw.json` carries the structural changes from Step 4. These files changed: [list `changed_files`]. Update ONLY the sections you own that are affected by these changes, and return ONLY what changed — unchanged sections are preserved by the patch merge. Use the 4-field contract (`problem_statement`, `evidence`, `root_cause`, `fix_direction`) when writing finding or pitfall entries. - The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. + The `changed_files` list MUST be expanded verbatim into the preamble (one path per line) — the Risk agent's recency sweep reads every file on it against the documented invariants and per-folder CLAUDE.md patterns (see its prompt body). An empty or summarized list silently disables that sweep. **If the list is no longer in context** (resume after `/compact`, `--continue`, or `--from 5`), regenerate it before dispatching — the baseline marker is untouched until Step 9's `save-baseline`, so this reproduces the Preamble's exact output: + + ```bash + python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" detect-changes + ``` **Output contract — append to each prompt, substituting that sub-agent's output path from the table as the "file path named above":** diff --git a/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md b/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md index fe4e15b5..b62a609d 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-8-cleanup.md @@ -12,11 +12,7 @@ TELEMETRY_STEP8_START=$(date -u +"%Y-%m-%dT%H:%M:%SZ") rm -f .archie/tmp/archie_sub*_$PROJECT_NAME.json .archie/tmp/archie_rules_$PROJECT_NAME.json .archie/tmp/archie_intent_prompt_${PROJECT_NAME}_*.txt .archie/tmp/archie_enrichment_${PROJECT_NAME}_*.json ``` -Remove artifacts from the retired drift step (projects upgraded from older Archie versions may still carry them; nothing reads these anymore): - -```bash -rm -f .archie/drift_report.json .archie/scan_report.md -``` +(Artifacts from the retired drift step — `drift_report.json`, `scan_report.md`, `drift_history/` — are swept by the installer's legacy cleanup at upgrade time, not here: that path is allowlisted, `$PROJECT_ROOT`-anchored, and runs for every workspace.) ```bash python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 8 diff --git a/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md b/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md index 8037b272..2966b05e 100644 --- a/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md +++ b/npm-package/assets/workflow/deep-scan/steps/step-9-finalize.md @@ -1,4 +1,4 @@ -## Step 9: Finalize — health metrics, baseline, telemetry +## Step 9: Finalize — health metrics, telemetry, baseline **Telemetry:** ```bash @@ -20,19 +20,7 @@ Save health scores to history for trending: python3 .archie/measure_health.py "$PROJECT_ROOT" --append-history --scan-type deep ``` -### Phase 2: Mark the run complete - -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 -``` - -Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): -```bash -python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE -``` -(Replace SCAN_MODE with the actual mode — "full" or "incremental") - -### Phase 3: Write telemetry +### Phase 2: Write telemetry Each prior step persisted its start timestamp to `.archie/telemetry/_current_run.json` via `telemetry.py mark` — so the final writer reads entirely from disk (no shell variables required, no /tmp timing file to assemble). This is what makes mid-run `/compact` safe: even if the orchestrator's conversation was compacted, every step's timing is on disk. @@ -55,6 +43,20 @@ python3 .archie/telemetry.py write "$PROJECT_ROOT" **Legacy fallback:** the old `.archie/tmp/archie_timing.json` + `telemetry.py --command … --timing-file …` invocation still works for any downstream tool that expects it, but the disk-persisted flow above is the compaction-safe canonical path. +### Phase 3: Mark the run complete + +Deliberately AFTER the telemetry flush: `complete-step 9` flips the run's `status` to `completed`, which tells `--continue` there is nothing left to resume. If the run is interrupted before this point, status stays `in_progress` and `--continue` simply re-runs Step 9 — every command in this step is idempotent. + +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" complete-step 9 +``` + +Save baseline marker for future incremental runs (use "full" or "incremental" based on SCAN_MODE): +```bash +python3 .archie/intent_layer.py deep-scan-state "$PROJECT_ROOT" save-baseline SCAN_MODE +``` +(Replace SCAN_MODE with the actual mode — "full" or "incremental") + ### Phase 4: Closing summary Present a short wrap-up to the user — a receipt, not a report (10 lines or fewer). State what the scan produced, with counts read via the allowlisted inspect commands (NEVER inline Python): @@ -63,8 +65,10 @@ Present a short wrap-up to the user — a receipt, not a report (10 lines or few python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" blueprint.json --query '.components.components|length' python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" rules.json --query '.rules|length' python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" findings.json --query '.findings|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" enrich_state.json --query '.done|length' +python3 .archie/intent_layer.py inspect "$PROJECT_ROOT" health.json --query .erosion ``` -Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (you know this count from Step 7), and findings tracked in `.archie/findings.json`. Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. +Cover: components discovered, enforcement rules generated, per-folder CLAUDE.md files created (the `enrich_state.json` query above — do not rely on remembering Step 7, a compact may have intervened), findings tracked in `.archie/findings.json`, and one health line (erosion score from `health.json`). Point the user at `{{COMMAND_PREFIX}}archie-viewer` for the full picture and `{{COMMAND_PREFIX}}archie-share` to publish it. If a count is unavailable (file missing, query returns nothing), omit that line rather than guessing. End with: **"Archie is now active. Architecture rules will be enforced on every code change. Run `{{COMMAND_PREFIX}}archie-deep-scan --incremental` after code changes to update the architecture analysis."** diff --git a/tests/test_asset_sync.py b/tests/test_asset_sync.py new file mode 100644 index 00000000..b3810fe0 --- /dev/null +++ b/tests/test_asset_sync.py @@ -0,0 +1,25 @@ +"""The npm-package mirror must stay byte-identical with canonical sources. + +scripts/verify_sync.py does the full-tree check (missing copies, orphan +assets, dead archie.mjs references) but used to be a manual pre-commit step — +a desync could land whenever the committer skipped it, and the wiring tests' +hardcoded spot-check can't catch orphans that exist in only one tree. +Running it under pytest makes the suite itself the gate. +""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_verify_sync_passes(): + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "verify_sync.py")], + capture_output=True, + text=True, + cwd=ROOT, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_comprehensive_wiring.py b/tests/test_comprehensive_wiring.py index 4c01049f..5b4946ee 100644 --- a/tests/test_comprehensive_wiring.py +++ b/tests/test_comprehensive_wiring.py @@ -56,6 +56,18 @@ def test_incremental_recency_sweep_is_wired(): assert "These files changed" in risk # the body names the preamble marker it keys on +def test_incremental_changed_files_survive_resume(): + """changed_files lives only in conversation memory (deliberately not + persisted) — so both the Wave 2 dispatch and the Resume Prelude must tell + the orchestrator to regenerate it via detect-changes after /compact, + --continue, or --from. Without this, the recency sweep silently no-ops on + every resumed incremental run.""" + wave2 = _read(STEPS / "step-5-wave2-reasoning.md") + assert "detect-changes" in wave2 and "no longer in context" in wave2 + prelude = _read(WF / "fragments" / "resume-prelude.md") + assert "detect-changes" in prelude and "SCAN_MODE=incremental" in prelude + + CONTRACT = "COMPREHENSIVE MODE — be exhaustive" @@ -87,7 +99,9 @@ def test_npm_package_workflow_in_sync(): """Spot-check: the npm-package mirror of the wired steps matches canonical.""" for rel in ("steps/step-1-scanner.md", "steps/step-6-rule-synthesis.md", "steps/step-5-wave2-reasoning.md", "steps/step-5b-risk.md", - "steps/step-9-finalize.md", "steps/step-3-wave1/orchestration.md"): + "steps/step-8-cleanup.md", "steps/step-9-finalize.md", + "steps/step-3-wave1/orchestration.md", + "fragments/resume-prelude.md"): canon = _read(WF / rel) mirror = _read(Path("npm-package/assets/workflow/deep-scan") / rel) assert canon == mirror, f"out of sync: {rel}" diff --git a/tests/test_detect_changes_ignore.py b/tests/test_detect_changes_ignore.py new file mode 100644 index 00000000..4d751c72 --- /dev/null +++ b/tests/test_detect_changes_ignore.py @@ -0,0 +1,69 @@ +"""detect-changes must honor .archieignore. + +`git diff --name-only` lists tracked files even when .archieignore'd (git only +knows .gitignore), so without filtering, vendored/generated paths would both +inflate the incremental threshold and get fed to the Risk agent's recency +sweep, which reads every listed file. The retired drift step filtered its +git-log list through the ignore system (filter-ignored); detect-changes is the +replacement source and must apply the same semantics. +""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +INTENT = str( + Path(__file__).resolve().parent.parent / "archie" / "standalone" / "intent_layer.py" +) + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run( + ["git", "-C", str(cwd), "-c", "commit.gpgsign=false", *args], + check=True, + capture_output=True, + ) + + +def test_detect_changes_filters_archieignored_paths(tmp_path): + _git(tmp_path, "init", "-q") + _git(tmp_path, "config", "user.email", "test@test.invalid") + _git(tmp_path, "config", "user.name", "test") + + (tmp_path / "src").mkdir() + (tmp_path / "vendor").mkdir() + (tmp_path / "src" / "app.py").write_text("print('v1')\n") + (tmp_path / "vendor" / "lib.py").write_text("print('v1')\n") + (tmp_path / ".archieignore").write_text("vendor/\n") + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-q", "-m", "baseline") + sha = subprocess.run( + ["git", "-C", str(tmp_path), "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, + ).stdout.strip() + + archie = tmp_path / ".archie" + archie.mkdir() + (archie / "last_deep_scan.json").write_text(json.dumps({"commit_sha": sha})) + (archie / "scan.json").write_text( + json.dumps({"file_tree": [{"path": f"f{i}.py"} for i in range(50)]}) + ) + + (tmp_path / "src" / "app.py").write_text("print('v2')\n") + (tmp_path / "vendor" / "lib.py").write_text("print('v2')\n") + # add only the source dirs — .archie/ tool state stays untracked, as in a + # default install (the generated .gitignore excludes tool internals) + _git(tmp_path, "add", "src", "vendor") + _git(tmp_path, "commit", "-q", "-m", "changes") + + out = subprocess.run( + [sys.executable, INTENT, "deep-scan-state", str(tmp_path), "detect-changes"], + capture_output=True, text=True, check=True, + ) + data = json.loads(out.stdout) + assert data["mode"] == "incremental" + # vendor/lib.py changed too, but it's .archieignore'd — only src/app.py + # may reach the changed list (and thus the recency sweep). + assert data["changed_files"] == ["src/app.py"] diff --git a/tests/test_install_loop.py b/tests/test_install_loop.py index ecf3af97..51e9ca2b 100644 --- a/tests/test_install_loop.py +++ b/tests/test_install_loop.py @@ -64,15 +64,35 @@ def test_install_removes_legacy_layout(tmp_path: Path) -> None: legacy_shared.mkdir(parents=True) (legacy_shared / "scope_resolution.md").write_text("OLD") + # Retired drift-step files from a pre-2.10 install (script + data + # artifacts) — the install sweep is the canonical retirement point. + legacy_archie = tmp_path / ".archie" + legacy_archie.mkdir(parents=True, exist_ok=True) + for retired in ( + "drift.py", "drift_report.json", "drift_diff.json", + "scan_report.md", "semantic_duplications.json", + ): + (legacy_archie / retired).write_text("OLD") + legacy_history = legacy_archie / "drift_history" + legacy_history.mkdir() + (legacy_history / "drift_20260101T000000.json").write_text("OLD") + install(tmp_path, ["claude"]) assert not (tmp_path / ".claude" / "skills" / "archie-deep-scan").exists() assert not (tmp_path / ".archie" / "prompts").exists() assert not (legacy_shared / "scope_resolution.md").exists() - # ...and the current layout is in place. + for retired in ( + "drift.py", "drift_report.json", "drift_diff.json", + "scan_report.md", "semantic_duplications.json", + ): + assert not (legacy_archie / retired).exists(), retired + assert not legacy_history.exists() + # ...and the current layout is in place (the sweep must not eat live files). assert ( tmp_path / ".archie" / "workflow" / "claude" / "deep-scan" / "SKILL.md" ).exists() + assert (tmp_path / ".archie" / "extract_output.py").exists() def test_install_sweeps_stale_command_shims_from_prior_version(tmp_path: Path) -> None: diff --git a/tests/test_upload.py b/tests/test_upload.py index 38544ec8..2b1dcfe6 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -78,6 +78,44 @@ def test_build_bundle_includes_all(mock_archie_dir): assert len(bundle["rules_proposed"]["rules"]) == 1 +def test_build_bundle_semantic_duplications_from_legacy_file(mock_archie_dir): + """The legacy semantic_duplications.json (Agent C output) still ships when + present — pre-upgrade shares keep their count until the installer sweeps + the file.""" + archie = mock_archie_dir / ".archie" + archie.joinpath("semantic_duplications.json").write_text(json.dumps({ + "duplications": [{"canonical": "loadUser", "twins": ["fetchUser"]}], + "scanned_at": "2026-01-01T00:00:00Z", + })) + bundle = build_bundle(mock_archie_dir) + assert bundle["semantic_duplications"] == [ + {"canonical": "loadUser", "twins": ["fetchUser"]} + ] + + +def test_build_bundle_semantic_duplications_structured_zero(mock_archie_dir): + """An explicit empty duplications list must still set the field — without + it the share viewer falls through to its prose heuristic, which can show a + phantom non-zero count.""" + archie = mock_archie_dir / ".archie" + archie.joinpath("semantic_duplications.json").write_text( + json.dumps({"duplications": []}) + ) + bundle = build_bundle(mock_archie_dir) + assert bundle["semantic_duplications"] == [] + + +def test_build_bundle_ignores_stale_drift_report(mock_archie_dir): + """drift_report.json is a retired artifact: its deep_findings used to be a + semantic_duplications source and must no longer feed the bundle.""" + archie = mock_archie_dir / ".archie" + archie.joinpath("drift_report.json").write_text(json.dumps({ + "deep_findings": [{"type": "semantic_duplication", "file": "a.py"}] + })) + bundle = build_bundle(mock_archie_dir) + assert "semantic_duplications" not in bundle + + def test_build_bundle_blueprint_only(tmp_path): archie = tmp_path / ".archie" archie.mkdir()