From 4ce8fc380994050d3d206f805fa2fd644223e41c Mon Sep 17 00:00:00 2001 From: gustavonline <8681157+gustavonline@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:47:05 +0200 Subject: [PATCH] Build Agentic Power BI System for issues #3 and #4 --- .../skills/agentic-powerbi-system/SKILL.md | 106 +++++ .agents/skills/fabric/SKILL.md | 9 +- .agents/skills/pbip/SKILL.md | 4 +- .agents/skills/powerbi/SKILL.md | 18 + .agents/skills/report/SKILL.md | 21 + .agents/skills/validation/SKILL.md | 21 +- .github/workflows/validate.yml | 3 +- .gitignore | 9 +- .npmignore | 6 + AGENTS.md | 69 ++- README.md | 88 ++-- .../agentic-power-bi-system-overview.svg | 6 +- docs/contract.md | 96 ++++ docs/validation.md | 75 +++ examples/.gitkeep | 1 + extensions/index.ts | 60 --- package.json | 51 +- prompts/audit-powerbi.md | 13 - prompts/plan-powerbi-page.md | 17 - prompts/validate-powerbi.md | 13 - tests/test-validator.mjs | 18 - workspace/PROJECT-BRIEF.md | 28 +- workspace/README.md | 20 + workspace/briefs/.gitkeep | 1 + workspace/engine/checks.py | 410 ++++++++++++++++ {scripts => workspace/engine}/doctor.mjs | 24 +- workspace/engine/seed-guard.mjs | 89 ++++ .../invalid/Broken.Report/definition.pbir | 0 .../valid/Repair.Report/definition.pbir | 0 .../Repair.Report/definition/report.json | 0 .../definition/model.tmdl | 0 workspace/engine/tests/test-validator.mjs | 207 +++++++++ workspace/engine/tests/test_template.py | 281 +++++++++++ workspace/engine/tracer.py | 436 ++++++++++++++++++ .../engine}/validate-pbip.mjs | 4 +- workspace/history/runs.jsonl | 1 + workspace/learning/.gitkeep | 1 + workspace/models/.gitkeep | 1 + workspace/reports/.gitkeep | 1 + workspace/runs/.gitkeep | 1 + workspace/state/.gitkeep | 1 + 41 files changed, 1986 insertions(+), 224 deletions(-) create mode 100644 .agents/skills/agentic-powerbi-system/SKILL.md create mode 100644 .npmignore rename assets/powerbi-template-overview.svg => docs/agentic-power-bi-system-overview.svg (95%) create mode 100644 docs/contract.md create mode 100644 docs/validation.md create mode 100644 examples/.gitkeep delete mode 100644 extensions/index.ts delete mode 100644 prompts/audit-powerbi.md delete mode 100644 prompts/plan-powerbi-page.md delete mode 100644 prompts/validate-powerbi.md delete mode 100644 tests/test-validator.mjs create mode 100644 workspace/README.md create mode 100644 workspace/briefs/.gitkeep create mode 100644 workspace/engine/checks.py rename {scripts => workspace/engine}/doctor.mjs (60%) create mode 100644 workspace/engine/seed-guard.mjs rename {tests => workspace/engine/tests}/fixtures/invalid/Broken.Report/definition.pbir (100%) rename {tests => workspace/engine/tests}/fixtures/valid/Repair.Report/definition.pbir (100%) rename {tests => workspace/engine/tests}/fixtures/valid/Repair.Report/definition/report.json (100%) rename {tests => workspace/engine/tests}/fixtures/valid/Repair.SemanticModel/definition/model.tmdl (100%) create mode 100644 workspace/engine/tests/test-validator.mjs create mode 100644 workspace/engine/tests/test_template.py create mode 100644 workspace/engine/tracer.py rename {scripts => workspace/engine}/validate-pbip.mjs (98%) create mode 100644 workspace/history/runs.jsonl create mode 100644 workspace/learning/.gitkeep create mode 100644 workspace/models/.gitkeep create mode 100644 workspace/reports/.gitkeep create mode 100644 workspace/runs/.gitkeep create mode 100644 workspace/state/.gitkeep diff --git a/.agents/skills/agentic-powerbi-system/SKILL.md b/.agents/skills/agentic-powerbi-system/SKILL.md new file mode 100644 index 0000000..3e47e93 --- /dev/null +++ b/.agents/skills/agentic-powerbi-system/SKILL.md @@ -0,0 +1,106 @@ +--- +name: agentic-powerbi-system +description: Primary System route for durable, decision-first Power BI work, specialist routing, and append-only proof. +--- + +# Agentic Power BI System primary route + +This is the one primary System skill. It preserves operational truth in +`workspace/` and uses the existing Power BI specialist skills for the work +they own. It is a filesystem workflow, not a service or orchestration runtime. + +## Route + +1. Read `workspace/PROJECT-BRIEF.md` and inspect relevant records in + `workspace/history/runs.jsonl` before selecting a route. Match the input + reference and route, not copied request text, to find the previous run. +2. Name the decision, audience, KPI contract, data owner, grain, freshness, + RLS owner, refresh/runtime boundary, publish authority, and proof needed. +3. Route the work to one or more specialist skills: + - `powerbi` for business question, decision, KPI definitions, and audit; + - `pbip` for PBIP/PBIR/TMDL structure and safe project edits; + - `dax` for explicit measures and semantic logic; + - `report` for page plans, report states, visuals, filters, and layout; + - `fabric` for approved service/workspace checks and publish operations; + - `validation` for deterministic local and optional native proof. +4. Create `workspace/runs//` and write structured `input.json`, + `output.json`, and `proof.json`. Failures write `failure.json`; recoveries + write `recovery.json` in their new run directory. +5. Append one JSON object to `workspace/history/runs.jsonl` with the run ID, + timestamps, status, input/output/proof references, previous-run relation, + and failure/recovery references. Do not rewrite an earlier record. +6. Promote an example only when the caller explicitly requests curation. The + promoted directory must contain its own `README.md` and `proof.json` and + must be understandable without importing this repository. + +Normal package creation runs the read-only seed guard. It permits the blank +seed placeholders and deliberate curated examples, but refuses to package a +non-empty ledger or mutable workspace evidence without changing that evidence. + +## Input contract + +The route accepts structured references: + +```text +route: audit | page-plan | validation | model | report | service | system-proof +input_ref: repository path, fixture reference, or approved work reference +decision_ref: workspace/PROJECT-BRIEF.md or a more specific brief reference +focus: optional bounded audit focus +business_question: required for a page-plan route +target: repository/project path for validation +promote_example: explicit true only when curation is desired +``` + +An input reference identifies the source; it is not a place to store a request +transcript, credentials, tenant identifiers, or private data. + +## Output and proof contract + +Every completed route returns or records: + +```text +status +run_id +previous_run_id +previous_run_relation +decision_ref +specialist_routes +output_ref +proof_ref +blockers +warnings +unavailable_checks +remaining_risks +``` + +An audit must cover semantic model structure, DAX and measure metadata, data +shaping risks, report storytelling, validation blockers/warnings, and next +actions. A page plan must return page title, audience and decision, required +measures/fields, KPI row, main visuals, slicers/filters, interpretation text, +and a validation checklist. A validation route must run deterministic checks +first, then report optional native checks only when installed, with blockers, +warnings, exact paths, unavailable checks, and commands run. + +## Power BI gates + +The route never substitutes a generated artifact for evidence. Keep each KPI's +name, DAX, format, interpretation, limitation, source, and business owner +together. Preserve star-schema grain, explicit relationships, hidden technical +keys, explicit measures, data ownership, RLS, refresh/runtime, workspace, and +publish authority. Ask before deleting or rebinding report assets, changing +`.platform` identity, publishing, overwriting service items, or changing +access/refresh policy. + +## Failure and recovery + +Keep a failed run and its evidence available. A recovery is a new run with +`previous_run_relation: recovery`, `previous_run_id` set to the unresolved +failed run, and a recovery reference. Recover a failed run at most once. An +ordinary continuation uses `predecessor`; the first relevant run uses `null`. + +## Reference implementation + +`workspace/engine/tracer.py` is the deterministic reference route. It proves +prior-run inspection, success, failure, recovery, append-only history, and +explicit example promotion with a fixed demonstration clock. It does not claim +that a live Power BI model, refresh, RLS, Fabric workspace, or publish exists. diff --git a/.agents/skills/fabric/SKILL.md b/.agents/skills/fabric/SKILL.md index 6adfa52..db65986 100644 --- a/.agents/skills/fabric/SKILL.md +++ b/.agents/skills/fabric/SKILL.md @@ -11,15 +11,10 @@ Use this skill when working with Power BI Service or Microsoft Fabric. ```bash fab --version -fab auth status -fab ls ``` -If not logged in, ask the user to run: - -```bash -fab auth login -``` +Only inspect an approved workspace or item after the user supplies that scope +and the operation is relevant. A version check alone is not service proof. ## Path format diff --git a/.agents/skills/pbip/SKILL.md b/.agents/skills/pbip/SKILL.md index dace56b..ca90563 100644 --- a/.agents/skills/pbip/SKILL.md +++ b/.agents/skills/pbip/SKILL.md @@ -37,7 +37,7 @@ Project/ 5. Validate immediately. ```bash -node scripts/validate-pbip.mjs . +node workspace/engine/validate-pbip.mjs . pbir validate "Report.Report" --all ``` @@ -49,7 +49,7 @@ When renaming tables, columns, measures, pages, or visuals, search across: - relationships - PBIR visual JSON - filters and sort definitions -- report extensions / visual calculations +- report metadata / visual calculations - DAX query files - diagram layouts diff --git a/.agents/skills/powerbi/SKILL.md b/.agents/skills/powerbi/SKILL.md index 0d3ca2d..d3ee002 100644 --- a/.agents/skills/powerbi/SKILL.md +++ b/.agents/skills/powerbi/SKILL.md @@ -42,6 +42,24 @@ Interpretation: Validation: ``` +## Harness-neutral audit contract + +Use this contract when the route is an audit. The input is an optional bounded +`focus` value; it is not a transcript. Return: + +```text +semantic model structure: +DAX measures and metadata: +Power Query/data shaping risks: +report design and storytelling: +validation blockers/warnings: +recommended next actions: +``` + +Include exact paths for findings and identify the owner or missing evidence for +each material risk. Keep the business question, decision, grain, KPI owner, +data boundary, and proof journey visible in the result. + ## Good Power BI habits - One page should answer one business question. diff --git a/.agents/skills/report/SKILL.md b/.agents/skills/report/SKILL.md index c4654c6..6d8178d 100644 --- a/.agents/skills/report/SKILL.md +++ b/.agents/skills/report/SKILL.md @@ -31,3 +31,24 @@ Use direct JSON editing only when the change is small and the structure is clear - Folder name and JSON `name` should match. - Ask before deleting pages/visuals or bulk formatting. - Validate after edits. + +## Harness-neutral page-plan contract + +Use this contract when planning a page from a `business_question` input. Return +these fields in order, with names that a caller can consume without a command +extension: + +```text +page title: +audience and decision: +required measures/fields: +KPI row: +main visuals: +slicers/filters: +business interpretation text: +validation checklist: +``` + +The page title must answer the business question, and the interpretation must +state the action or decision supported by the evidence. Do not claim data, +refresh, or visual validation that was not run. diff --git a/.agents/skills/validation/SKILL.md b/.agents/skills/validation/SKILL.md index f575865..437789e 100644 --- a/.agents/skills/validation/SKILL.md +++ b/.agents/skills/validation/SKILL.md @@ -12,7 +12,7 @@ Use this before saying a Power BI task is done. 1. Local structural validation: ```bash -node scripts/validate-pbip.mjs . +node workspace/engine/validate-pbip.mjs . ``` 2. PBIR validation when `pbir` is available: @@ -24,10 +24,12 @@ pbir validate "Report.Report" --all 3. Fabric/service checks when relevant: ```bash -fab auth status fab exists "Workspace.Workspace/Model.SemanticModel" ``` +Run service checks only for an approved target. Do not turn a missing optional +tool or an unrun service check into a completion claim. + 4. Rename cascade checks: ```bash @@ -44,6 +46,21 @@ Files changed: Remaining risks: ``` +## Harness-neutral validation contract + +The validation route accepts a repository or project `target` (default `.`) +and runs these steps in order: + +1. `node workspace/engine/validate-pbip.mjs `; +2. `pbir validate --all` for each report when `pbir` is + installed; +3. approved Fabric existence or item checks only when service scope is + relevant. + +Return blockers, warnings, exact paths, commands run, unavailable checks, and +remaining risks. Native Power BI, Fabric, Tabular Editor, and DAX Studio +results are optional evidence and must never be invented. + ## Common blockers - invalid JSON diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index a086ac1..a62853e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,4 +1,4 @@ -name: Validate template +name: Validate Agentic Power BI System on: push: @@ -16,5 +16,6 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 20 + - run: npm run doctor - run: npm run validate - run: npm test diff --git a/.gitignore b/.gitignore index 18a434b..5e575a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ node_modules/ +__pycache__/ +*.py[cod] +.pytest_cache/ .DS_Store Thumbs.db .env .env.* !.env.example -.pi/tmp/ -.pi/sessions/ +.pbi/ +*.pbix +workspace/state/* +!workspace/state/.gitkeep diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..b8a7bdc --- /dev/null +++ b/.npmignore @@ -0,0 +1,6 @@ +__pycache__ +**/__pycache__ +*.pyc +**/*.pyc +.pbi/ +*.pbix diff --git a/AGENTS.md b/AGENTS.md index 382f9aa..6dfc88b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,51 +1,78 @@ -# PowerBI-template +# Agentic Power BI System -Turn a business question into an owned, validated Power BI model and report. +Turn a business question into an owned, validated Power BI model, report, and +decision journey that can retain its operational proof across runs. + +## System contract + +This repository is the standalone Agentic Power BI System. It follows the +System Template contract: `workspace/` is persistent operational truth, +`examples/` contains only deliberate standalone proof, and `docs/` contains +the public contract and validation recipe. The System is self-contained and is +not installed into another agent runtime. + +The primary route is `.agents/skills/agentic-powerbi-system/SKILL.md`. It first +inspects relevant records in `workspace/history/runs.jsonl`, routes Power BI +work through the specialist skills, and preserves input, output, proof, +failure, recovery, and previous-run references. Request content is represented +by references and structured facts, never by transcript files. Do not put +secrets, tenant identifiers, private data, or generated local Power BI state in +the repository. ## Start -1. Read `workspace/PROJECT-BRIEF.md` and preserve resolved AIOS or project context. +1. Read `workspace/PROJECT-BRIEF.md` and preserve any resolved business or + project context. 2. Name the decision, audience, KPI definitions, data owners, refresh/runtime, proof, and optional Design-template handoff. 3. Model first: use a star schema, explicit relationships, hidden technical keys, and explicit measures. 4. Report second: one page per business question, with the decision and its evidence ahead of decoration. -5. Validate every PBIP/PBIR/TMDL edit. +5. Validate every PBIP/PBIR/TMDL edit and record unavailable optional tools. -## Routes +## Specialist routes -- `.agents/skills/powerbi/SKILL.md`: business analytics and decision framing -- `.agents/skills/pbip/SKILL.md`: PBIP/PBIR/TMDL structure +- `.agents/skills/powerbi/SKILL.md`: business question, decision, KPI, and + audit framing +- `.agents/skills/pbip/SKILL.md`: PBIP/PBIR/TMDL structure and safe edits - `.agents/skills/dax/SKILL.md`: measures and semantic logic -- `.agents/skills/report/SKILL.md`: report composition, states, and filters -- `.agents/skills/fabric/SKILL.md`: service and Fabric work -- `.agents/skills/validation/SKILL.md`: completion proof +- `.agents/skills/report/SKILL.md`: page plans, states, visuals, and filters +- `.agents/skills/fabric/SKILL.md`: service and Fabric work under authority + gates +- `.agents/skills/validation/SKILL.md`: deterministic completion proof -Pi is optional. Use `scripts/doctor.mjs` and `scripts/validate-pbip.mjs` -directly in any harness. +The specialist skills are harness-neutral contracts. The deterministic doctor, +PBIP validator, Python System engine, and tests live under +`workspace/engine/` and are callable directly or through the package scripts. +Normal `npm pack`/publish packaging also runs the read-only seed guard; mutable +workspace evidence causes an explicit failure and is never removed. ## Rules - Keep KPI name, DAX, format, interpretation, limitation, and business owner together. - Use deterministic validation before visual inspection or guesses. -- Preserve data ownership, row-level security, refresh, and workspace - boundaries. -- Keep secrets, tenant identifiers, private data, and generated local Power BI - state out of Git. -- Ask before deleting pages or visuals, rebinding, publishing, overwriting - service items, changing `.platform` IDs, or changing access/refresh policy. +- Preserve data ownership, row-level security, refresh, workspace, and publish + authority boundaries. +- Ask before deleting pages or visuals, rebinding reports, changing `.platform` + IDs, publishing, overwriting service items, or changing access/refresh + policy. +- Do not invent PBIX/PBIP evidence or require optional Power BI tools that are + unavailable. - Finish with changed files, validation evidence, unavailable checks, and remaining risk. ## Proof ```sh -node scripts/doctor.mjs -node scripts/validate-pbip.mjs . --ignore-tests +node workspace/engine/doctor.mjs +node workspace/engine/validate-pbip.mjs . --ignore-tests npm test +npm pack --dry-run ``` Use `pbir validate`, Fabric CLI, Tabular Editor, DAX Studio, Power BI Desktop, -or `pbi-tools` only when the project and available environment justify them. +or `pbi-tools` only when the command is truthfully installed and the project +scope justifies it. Record exact unavailable checks rather than inferring +their result. diff --git a/README.md b/README.md index 609fd2c..3594f79 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,51 @@ -![PowerBI-template workflow](assets/powerbi-template-overview.svg) +![Agentic Power BI System workflow](docs/agentic-power-bi-system-overview.svg) -# PowerBI-template +# Agentic Power BI System -A private foundation for turning a business question into an owned Power BI -model, report, and decision workflow. +A standalone System for turning a business question into an owned, validated +Power BI model, report, and decision workflow with durable proof. ```text business question → definitions and data → model → report → decision → proof ``` -Start from AIOS context, a standalone brief, or an approved Design-template -handoff. This repository owns the Power BI solution after the handoff. +Start from the project brief, resolved business context, or an approved +Design-template handoff. This repository owns the Power BI solution after the +handoff while preserving the decision, data, security, refresh, and publish +boundaries. ## Start 1. Complete [workspace/PROJECT-BRIEF.md](workspace/PROJECT-BRIEF.md). -2. Keep business definitions next to their measure, format, interpretation, and +2. Read the primary route at + `.agents/skills/agentic-powerbi-system/SKILL.md`. +3. Keep business definitions next to their measure, format, interpretation, and limitation. -3. Build the model before the report. -4. Make each report page answer one named business question. -5. Validate before opening, publishing, or claiming completion. +4. Build the model before the report. +5. Make each report page answer one named business question. +6. Validate before opening, publishing, or claiming completion. ```sh npm run doctor npm run validate npm test +npm pack --dry-run ``` ## What is included -- Project-local skills for Power BI, PBIP/PBIR/TMDL, DAX, reports, Fabric, and +- One primary System skill for durable routing and run evidence. +- Specialist skills for Power BI, PBIP/PBIR/TMDL, DAX, reports, Fabric, and validation. -- Short prompts for audit, page planning, and validation. - A dependency-free PBIP/PBIR/TMDL validator and valid/invalid fixtures. -- An optional Pi extension that exposes doctor and validation commands. - -Pi is an adapter, not the primary interface. Any coding-agent harness can use -the repository instructions, skills, prompts, and Node scripts directly. +- A standard-library tracer for success, failure, recovery, and deliberate + example promotion. ## Workflow | Need | Route | | --- | --- | +| System routing, prior runs, and evidence | `.agents/skills/agentic-powerbi-system/SKILL.md` | | Business question, KPI, and decision | `.agents/skills/powerbi/SKILL.md` | | PBIP/PBIR/TMDL structure and safe edits | `.agents/skills/pbip/SKILL.md` | | Measures and semantic logic | `.agents/skills/dax/SKILL.md` | @@ -51,26 +55,56 @@ the repository instructions, skills, prompts, and Node scripts directly. An approved Design-template handoff can supply `DESIGN.md`, report composition, tokens, and preview intent. Revalidate it against actual Power BI capabilities, -data, accessibility, and tenant constraints; Design-template is not a runtime -dependency. - -## Optional Pi adapter +data, accessibility, and tenant constraints; the handoff is reference input, +not an execution requirement. -When Pi is useful, install the package project-locally: +## Persistent workspace -```sh -pi install -l git:github.com/onlinesourdough/PowerBI-template +```text +workspace/ +├── PROJECT-BRIEF.md decision, KPI, data, and proof contract +├── briefs/ active bounded briefs +├── models/ active semantic-model work +├── reports/ active report work +├── state/ local System state +├── runs/ one durable evidence directory per route +├── history/runs.jsonl append-only relation between runs +├── learning/ durable notes intentionally retained +└── engine/ optional local validators, tracer, and tests +examples/ deliberately curated standalone proof +docs/ contract and validation notes ``` -It provides `/powerbi-doctor`, `/powerbi-validate`, prompt templates, and edit -warnings. Without Pi, use the same scripts directly. +The primary route inspects prior records before routing work. It never copies +request transcripts into the ledger. A run is valid without promotion; a +curated example is written only when explicitly requested and includes its own +README and proof file. ## Safety Ask before deleting pages or visuals, rebinding reports, changing `.platform` IDs, publishing, overwriting service items, or changing refresh and access -policy. Keep tenant IDs, credentials, connection strings, and private data out -of Git and fixtures. +policy. Keep tenant IDs, credentials, connection strings, private data, and +generated local Power BI state out of Git and fixtures. + +`npm pack --dry-run` runs a read-only seed guard. It succeeds for blank +placeholders and deliberate curated examples, and refuses to bundle mutable +workspace evidence without deleting or resetting it. + +## Deterministic System proof + +Run the full local route in a disposable checkout or temporary copy: + +```sh +python3 workspace/engine/tracer.py --promote-example +python3 workspace/engine/tracer.py --simulate-failure +python3 workspace/engine/tracer.py --recover --promote-example +``` + +The fixed demonstration clock makes the first clean run `run-0001`. The failed +run remains in the ledger, and the recovery run points to that predecessor. +The source repository should remain a seed with empty operational placeholders +after validation. ## License diff --git a/assets/powerbi-template-overview.svg b/docs/agentic-power-bi-system-overview.svg similarity index 95% rename from assets/powerbi-template-overview.svg rename to docs/agentic-power-bi-system-overview.svg index fc61912..bea133b 100644 --- a/assets/powerbi-template-overview.svg +++ b/docs/agentic-power-bi-system-overview.svg @@ -1,8 +1,8 @@ - PowerBI-template workflow + Agentic Power BI System workflow A business question moves through definitions and data, model, report, decision, and proof. - ONLINESOURDOUGH · POWERBI-TEMPLATE + ONLINESOURDOUGH · AGENTIC POWER BI SYSTEM Bake the report around the decision. QUESTIONWhat changes?Audience, owner,decision and proof. @@ -15,6 +15,6 @@ - Optional Design-template handoff in · measured business learning back to AIOS + Optional Design-template handoff in · measured business learning back to the owner No chart buffet. Serve the decision. diff --git a/docs/contract.md b/docs/contract.md new file mode 100644 index 0000000..93c6009 --- /dev/null +++ b/docs/contract.md @@ -0,0 +1,96 @@ +# Agentic Power BI System contract + +## Product and boundary + +The canonical product name is **Agentic Power BI System**. This repository is a +standalone System mapped to the System Template contract. It owns its local +briefs, model/report work, state, run evidence, history, and learning under +`workspace/`. + +The first-class concepts remain Space, System, and Project. This repository is +the System capability; a bounded Power BI delivery is a Project outcome inside +its brief and workspace. The repository does not import another System, +centralize records, or require a service to interpret its contract. + +## Functional roots + +Only these visible functional roots are used: + +| Path | Purpose | +| --- | --- | +| `workspace/` | persistent briefs, model/report work, state, and operational truth | +| `examples/` | deliberately curated standalone proof | +| `docs/` | public contract and validation notes | + +Technical implementation lives below `workspace/engine/`. Root instructions, +the README, the primary System skill, package metadata, and hidden CI are shell +support rather than additional functional roots. + +## Primary route + +`.agents/skills/agentic-powerbi-system/SKILL.md` is the one primary System skill. +It must: + +1. inspect relevant prior records before routing work; +2. select the existing Power BI specialist route(s) for the business question, + PBIP/PBIR/TMDL structure, DAX, report composition, Fabric, or validation; +3. write structured input, output, and proof references under + `workspace/runs//`; +4. append one JSON object to `workspace/history/runs.jsonl`; +5. preserve failure and recovery evidence without rewriting the failed record; +6. promote an example only after an explicit curation choice. + +Inputs are references and structured facts, not request transcripts. Nothing +in the route may invent PBIX/PBIP, refresh, access, tenant, or publish proof. + +## Run ledger + +Each ledger object contains: + +```text +run_id +started_at +finished_at +status +input_ref +output_ref +proof_ref +previous_run_id +previous_run_relation +failure +recovery +``` + +`previous_run_relation` is `null` for the first relevant run, `predecessor` +for ordinary continuation, and `recovery` when the route explicitly recovers +an unresolved failed predecessor. A failed record stays append-only and can be +recovered at most once. References point to the run directory; the ledger is +not a second content database. + +## Power BI authority gates + +The System preserves these gates in the brief and specialist routes: + +- KPI definition, interpretation, limitation, and business owner; +- canonical data ownership, grain, relationships, freshness, and privacy; +- row-level security design and test ownership; +- refresh, gateway/capacity, workspace, and operational ownership; +- publish, overwrite, access, sensitivity, and rollback authority. + +Optional native Power BI, PBIR, PBIP, Fabric, or Tabular Editor checks are +reported only when their commands are installed and the project scope permits +them. + +## Promotion boundary + +`examples/` is not a scratch area. A promoted leaf must contain its own +`README.md` and `proof.json`, identify its source run, carry a curated marker, +and remain understandable without importing this repository. + +## Seed packaging boundary + +The package whitelist contains the System seed, not active operational state. +The `prepack` guard allows blank `.gitkeep` placeholders and a blank +`workspace/history/runs.jsonl`. It refuses non-empty history and mutable files +under the workspace operational directories. It does not inspect or remove +curated examples, and a refusal leaves all evidence in place. diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..8e169c4 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,75 @@ +# Validation + +Validation is deterministic first and tool-assisted only when the tool is +truthfully available. + +## Local proof + +From the repository root: + +```sh +node workspace/engine/doctor.mjs +node workspace/engine/validate-pbip.mjs . --ignore-tests +npm test +npm pack --dry-run +``` + +The doctor requires Node.js, Git, and Python for the local engine. Fabric, +`pbir`, `pbi-tools`, Tabular Editor, and other native Power BI tools are +optional. A missing optional command is an unavailable check, not a failure +that can be filled by assumption. + +The package prepack guard allows blank `.gitkeep` seed placeholders, the blank +`workspace/history/runs.jsonl`, and deliberate curated examples. It refuses to +package non-empty run history or mutable workspace directories and leaves that +operational evidence untouched. + +When a Power BI project is present, run the deterministic validator first. If +`pbir` is installed, run: + +```sh +pbir validate "Report.Report" --all +``` + +Use Fabric or workspace checks only with an approved target and authority. Do +not run an authentication-status check as a substitute for publish proof. + +## System route proof + +Use a disposable checkout or temporary copy so operational artifacts do not +become seed content: + +```sh +python3 workspace/engine/tracer.py --promote-example +python3 workspace/engine/tracer.py --simulate-failure +python3 workspace/engine/tracer.py --recover --promote-example +``` + +The deterministic demonstration records a successful run, a recoverable failed +run, and a recovery whose `previous_run_id` points to the failed predecessor. +The explicit promotion creates a self-contained example with `README.md` and +`proof.json`. + +## Clean-copy recipe + +For an independent local copy, copy the checkout to a temporary directory and +run the commands above from that copy. A local copy is sufficient; no remote +or service is contacted by the structural checks, validator, tests, package +dry-run, or tracer. + +## Validation report + +Record: + +```text +Validation run: +Blockers: +Warnings: +Files changed: +Unavailable checks: +Remaining risks: +``` + +Never claim service refresh, RLS, publish, or native Power BI validation unless +the corresponding evidence exists and its owner and boundary are recorded in +the brief. diff --git a/examples/.gitkeep b/examples/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/.gitkeep @@ -0,0 +1 @@ + diff --git a/extensions/index.ts b/extensions/index.ts deleted file mode 100644 index e2ad3d8..0000000 --- a/extensions/index.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; -import path from "node:path"; - -function root() { - return path.resolve(__dirname, ".."); -} - -function needsPowerBiValidation(filePath: unknown) { - if (typeof filePath !== "string") return false; - const p = filePath.replace(/\\/g, "/"); - return p.endsWith(".tmdl") || p.endsWith(".pbir") || (p.endsWith(".json") && p.includes(".Report/")); -} - -export default function (pi: ExtensionAPI) { - const doctorScript = path.join(root(), "scripts", "doctor.mjs"); - const validateScript = path.join(root(), "scripts", "validate-pbip.mjs"); - - pi.registerCommand("powerbi-doctor", { - description: "Check optional local Power BI tooling", - handler: async (_args, ctx) => { - const result = await pi.exec("node", [doctorScript], { timeout: 30_000 }); - const output = `${result.stdout || ""}${result.stderr || ""}`.trim() || "No output"; - if (ctx.hasUI) ctx.ui.notify("Power BI doctor finished", result.code === 0 ? "info" : "warning"); - pi.sendMessage({ customType: "powerbi-doctor", content: output, display: true, details: { code: result.code } }); - }, - }); - - pi.registerCommand("powerbi-validate", { - description: "Validate PBIP/PBIR/TMDL files in this project", - handler: async (args, ctx) => { - const target = args?.trim() || "."; - const result = await pi.exec("node", [validateScript, target], { timeout: 60_000 }); - const output = `${result.stdout || ""}${result.stderr || ""}`.trim() || "No output"; - const level = result.code === 0 ? "info" : result.code === 1 ? "warning" : "error"; - if (ctx.hasUI) ctx.ui.notify("Power BI validation finished", level); - pi.sendMessage({ customType: "powerbi-validation", content: output, display: true, details: { code: result.code, target } }); - }, - }); - - pi.on("tool_result", async (event) => { - if (event.toolName !== "write" && event.toolName !== "edit") return undefined; - const filePath = (event.input as { path?: unknown }).path; - if (!needsPowerBiValidation(filePath)) return undefined; - - const result = await pi.exec("node", [validateScript, "--changed", String(filePath)], { timeout: 30_000 }); - if (result.code === 0) return undefined; - - const output = `${result.stdout || ""}${result.stderr || ""}`.trim(); - return { - isError: result.code === 2, - content: [ - { - type: "text", - text: `Power BI validation ${result.code === 2 ? "failed" : "warning"} for ${filePath}\n\n${output}`, - }, - ], - details: { code: result.code, filePath }, - }; - }); -} diff --git a/package.json b/package.json index 4187b5d..e70a761 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { - "name": "powerbi-template", + "name": "agentic-powerbi-system", "version": "0.2.0", - "description": "Project-local context, skills, and validation for owned Power BI solutions.", + "description": "Agentic Power BI System: a standalone, decision-first Power BI workflow with durable proof.", "keywords": [ - "pi-package", + "agentic-system", "agent-skills", "power-bi", "pbip", @@ -13,21 +13,40 @@ "business-analytics" ], "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/onlinesourdough/PowerBI-template.git" - }, - "pi": { - "extensions": ["extensions/index.ts"], - "skills": [".agents/skills"], - "prompts": ["prompts"] - }, + "files": [ + "AGENTS.md", + "README.md", + "LICENSE", + ".agents/skills", + "docs", + "examples", + "workspace/PROJECT-BRIEF.md", + "workspace/README.md", + "workspace/briefs", + "workspace/models", + "workspace/reports", + "workspace/state", + "workspace/runs", + "workspace/history/runs.jsonl", + "workspace/learning", + "workspace/engine/checks.py", + "workspace/engine/doctor.mjs", + "workspace/engine/seed-guard.mjs", + "workspace/engine/tracer.py", + "workspace/engine/validate-pbip.mjs", + "workspace/engine/tests/test_template.py", + "workspace/engine/tests/test-validator.mjs", + "workspace/engine/tests/fixtures" + ], "scripts": { - "doctor": "node scripts/doctor.mjs", - "validate": "node scripts/validate-pbip.mjs . --ignore-tests", - "test": "node tests/test-validator.mjs" + "doctor": "node workspace/engine/doctor.mjs", + "validate": "node workspace/engine/validate-pbip.mjs . --ignore-tests", + "test": "node workspace/engine/tests/test-validator.mjs && python3 -m unittest discover -s workspace/engine/tests -p 'test_*.py'", + "trace": "python3 workspace/engine/tracer.py --promote-example", + "prepack": "node workspace/engine/seed-guard.mjs" }, "engines": { - "node": ">=20" + "node": ">=20", + "python": ">=3.9" } } diff --git a/prompts/audit-powerbi.md b/prompts/audit-powerbi.md deleted file mode 100644 index 9eebd2b..0000000 --- a/prompts/audit-powerbi.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -description: Audit a Power BI project for model, DAX, report design, and validation readiness -argument-hint: "[focus]" ---- -Audit this Power BI project. Focus: `$ARGUMENTS`. - -Cover: -- semantic model structure -- DAX measures and metadata -- Power Query/data shaping risks -- report design and storytelling -- validation blockers/warnings -- recommended next actions diff --git a/prompts/plan-powerbi-page.md b/prompts/plan-powerbi-page.md deleted file mode 100644 index 7b91925..0000000 --- a/prompts/plan-powerbi-page.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -description: Plan a Power BI report page from a business question -argument-hint: "" ---- -Plan a Power BI report page for: - -`$ARGUMENTS` - -Return: -- page title -- audience and decision -- required measures/fields -- KPI row -- main visuals -- slicers/filters -- business interpretation text -- validation checklist diff --git a/prompts/validate-powerbi.md b/prompts/validate-powerbi.md deleted file mode 100644 index cc63924..0000000 --- a/prompts/validate-powerbi.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -description: Validate the current Power BI project -argument-hint: "[path]" ---- -Validate the Power BI project at `${1:-.}`. - -Use deterministic checks first: - -1. `node scripts/validate-pbip.mjs ${1:-.}` if available. -2. `pbir validate --all` for each report if `pbir` is available. -3. `fab auth status` / `fab exists` only if service bindings are relevant. - -Return blockers, warnings, exact paths, and commands run. diff --git a/tests/test-validator.mjs b/tests/test-validator.mjs deleted file mode 100644 index d7c1d08..0000000 --- a/tests/test-validator.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { spawnSync } from "node:child_process"; - -const cases = [ - { path: "tests/fixtures/valid", expected: 0 }, - { path: "tests/fixtures/invalid", expected: 2 }, -]; - -for (const testCase of cases) { - const result = spawnSync(process.execPath, ["scripts/validate-pbip.mjs", testCase.path], { - encoding: "utf8", - }); - if (result.status !== testCase.expected) { - process.stderr.write(`Validator case ${testCase.path} returned ${result.status}; expected ${testCase.expected}.\n${result.stdout}${result.stderr}`); - process.exit(1); - } -} - -process.stdout.write("PBIP validator fixtures: PASS\n"); diff --git a/workspace/PROJECT-BRIEF.md b/workspace/PROJECT-BRIEF.md index 74e8778..6f78124 100644 --- a/workspace/PROJECT-BRIEF.md +++ b/workspace/PROJECT-BRIEF.md @@ -1,18 +1,28 @@ -# Power BI project brief +# Agentic Power BI System project brief + +This file is the active decision contract for one bounded Power BI outcome. Do +not fill unknowns with invented data or service evidence. - **Business question:** What must the report answer? - **Decision:** What will someone do differently with the answer? -- **Audience and owner:** Who decides, and who owns the result? -- **KPIs:** Name, definition, source, owner, format, target, interpretation, and - known limitation. +- **Audience and owner:** Who decides, who uses the result, and who owns it? +- **KPIs:** For each KPI record the name, definition, source, data owner, + format, target, interpretation, limitation, and approval status. - **Data:** Canonical systems, tables, grain, relationships, freshness, quality, privacy, and access boundaries. -- **Refresh and runtime:** Desktop, gateway, workspace, schedule, capacity, and - operational owner. +- **Model:** Star-schema facts and dimensions, explicit relationships, hidden + technical keys, and explicit measures. +- **Security:** Row-level security roles, filter logic, owner, test evidence, + and unresolved access risks. +- **Refresh and runtime:** Desktop, gateway, workspace, schedule, capacity, + credentials/connection ownership, freshness expectation, and operational + owner. Keep credentials and connection details out of this file. - **Report behavior:** Pages, filters, drill paths, loading/no-data/error/stale states, accessibility, and mobile requirements. +- **Publish authority:** Approved workspace/item, promotion authority, overwrite + boundary, sensitivity/access review, and rollback path. - **Design handoff:** Optional path to an approved Design-template bundle and the - decisions it settles. -- **Proof:** The real question, validation, refresh, access, and decision journey - that must pass. + decisions it settles; the handoff does not replace validation. +- **Proof:** The real question, model validation, refresh, access, publish, and + decision journey that must pass. - **Non-goals:** What the first complete result deliberately excludes. diff --git a/workspace/README.md b/workspace/README.md new file mode 100644 index 0000000..2a04e87 --- /dev/null +++ b/workspace/README.md @@ -0,0 +1,20 @@ +# Workspace + +`workspace/` is the persistent operational truth for the Agentic Power BI +System. It is a local filesystem contract, not a service or a shared store. + +- `PROJECT-BRIEF.md` is the active decision, KPI, data, security, refresh, and + publish contract. +- `briefs/` holds additional bounded briefs when a project needs them. +- `models/` and `reports/` hold active semantic-model and report work. +- `state/` holds only safe, local System state; generated Power BI cache remains + excluded. +- `runs/` holds durable evidence for each route attempt. +- `history/runs.jsonl` keeps the append-only relation between runs. +- `learning/` holds durable notes intentionally retained for future work. +- `engine/` contains optional local validators, the deterministic tracer, and + dependency-light tests. + +Run records keep references to input, output, and proof artifacts instead of +copying request transcripts into the ledger. No service, database, or external +System installation is needed to understand this workspace. diff --git a/workspace/briefs/.gitkeep b/workspace/briefs/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/briefs/.gitkeep @@ -0,0 +1 @@ + diff --git a/workspace/engine/checks.py b/workspace/engine/checks.py new file mode 100644 index 0000000..3d93d76 --- /dev/null +++ b/workspace/engine/checks.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +"""Structural, contract, and stale-route checks for the standalone System.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + + +PRODUCT_NAME = "Agentic Power BI System" +VISIBLE_FUNCTIONAL_ROOTS = {"workspace", "examples", "docs"} +PUBLIC_ROOT_FILES = {"AGENTS.md", "README.md", "LICENSE"} +TOOLCHAIN_ROOT_FILES = { + "Cargo.lock", + "Cargo.toml", + "Gemfile", + "GNUmakefile", + "Makefile", + "go.mod", + "go.sum", + "package-lock.json", + "package.json", + "pnpm-lock.yaml", + "pyproject.toml", + "pytest.ini", + "setup.cfg", + "tox.ini", + "yarn.lock", +} +REQUIRED_PATHS = ( + "AGENTS.md", + "README.md", + ".agents/skills/agentic-powerbi-system/SKILL.md", + ".agents/skills/powerbi/SKILL.md", + ".agents/skills/pbip/SKILL.md", + ".agents/skills/dax/SKILL.md", + ".agents/skills/report/SKILL.md", + ".agents/skills/fabric/SKILL.md", + ".agents/skills/validation/SKILL.md", + "workspace/PROJECT-BRIEF.md", + "workspace/README.md", + "workspace/briefs", + "workspace/models", + "workspace/reports", + "workspace/state", + "workspace/runs", + "workspace/history/runs.jsonl", + "workspace/learning", + "workspace/engine/tracer.py", + "workspace/engine/checks.py", + "workspace/engine/doctor.mjs", + "workspace/engine/seed-guard.mjs", + "workspace/engine/validate-pbip.mjs", + "workspace/engine/tests/test_template.py", + "workspace/engine/tests/test-validator.mjs", + "docs/contract.md", + "docs/validation.md", +) +STALE_ROOT_NAMES = { + "assets", + "engine", + "ext" + "ensions", + "prom" + "pts", + "scripts", + "tests", +} +CURATED_EXAMPLE_FIELDS = {"curated", "example", "source_run_id", "status"} +LEDGER_FIELDS = { + "run_id", + "started_at", + "finished_at", + "status", + "input_ref", + "output_ref", + "proof_ref", + "previous_run_id", + "previous_run_relation", + "failure", + "recovery", +} +TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + + +def repository_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _public_text_files(root: Path) -> Iterable[Path]: + """Yield source files that form the public or operational contract.""" + + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + if ".git" in path.parts or "node_modules" in path.parts: + continue + if "__pycache__" in path.parts or path.suffix in {".pyc", ".pbix"}: + continue + yield path + + +def _contains_stale_language(text: str) -> List[str]: + """Return forbidden legacy route or unsafe public wording fragments.""" + + # Keep legacy fragments assembled so this checker does not become a stale + # public artifact containing the routes it is designed to catch. + forbidden = ( + "PowerBI-" + "template", + "prom" + "pts/", + "p" + "i" + ".prom" + "pts", + "." + "p" + "i" + "/", + "@mario" + "zechner", + "hand" + "off " + "sche" + "ma", + "hand" + "off-" + "sche" + "ma", + "hand" + " " + "off " + "sche" + "ma", + "hand" + " " + "off-" + "sche" + "ma", + "shared " + "sche" + "ma", + "shared-" + "sche" + "ma", + "cross-" + "system " + "sche" + "ma", + "cross " + "system " + "sche" + "ma", + "runtime " + "dependency", + "runtime-" + "dependency", + "shared " + "package", + "central " + "database", + "raw " + "pr" + "ompt", + "Agentic " + "Project " + "Template", + ) + lowered = text.lower() + return [phrase for phrase in forbidden if phrase.lower() in lowered] + + +def _relative_reference(root: Path, value: object) -> Optional[Path]: + if not isinstance(value, str) or not value: + return None + candidate = (root / value).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError: + return None + return candidate + + +def check_structure(root: Path) -> List[str]: + """Return structural violations; an empty list means the shell is valid.""" + + errors: List[str] = [] + if not root.is_dir(): + return [f"repository root is not a directory: {root}"] + + visible_entries = [entry for entry in root.iterdir() if not entry.name.startswith(".")] + for entry in visible_entries: + if entry.name in VISIBLE_FUNCTIONAL_ROOTS: + continue + if entry.name in PUBLIC_ROOT_FILES or entry.name in TOOLCHAIN_ROOT_FILES: + continue + if entry.name in STALE_ROOT_NAMES: + errors.append(f"legacy root path is present: {entry.name}/") + elif entry.is_dir() or entry.is_symlink(): + errors.append(f"visible functional root is not allowed: {entry.name}/") + else: + errors.append(f"visible root file is not part of the public shell: {entry.name}") + + for stale_name in sorted(STALE_ROOT_NAMES): + if (root / stale_name).exists(): + errors.append(f"legacy root path is present: {stale_name}/") + + hidden_legacy = {"." + "p" + "i", "p" + "i" + ".prom" + "pts"} + for path in root.rglob("*"): + if ".git" in path.parts: + continue + if any(part in hidden_legacy for part in path.parts): + errors.append(f"legacy hidden path is present: {path.relative_to(root)}") + + for relative in REQUIRED_PATHS: + if not (root / relative).exists(): + errors.append(f"required path is missing: {relative}") + + for path in (root / "workspace", root / "examples", root / "docs"): + if path.is_symlink(): + errors.append(f"functional root must be a directory, not a symlink: {path.name}/") + elif not path.is_dir(): + errors.append(f"functional root must be a directory: {path.name}/") + + history = root / "workspace" / "history" / "runs.jsonl" + if history.exists() and not history.is_file(): + errors.append("workspace/history/runs.jsonl must be a file") + if history.is_file(): + _check_ledger(history, root, errors) + + for path in _public_text_files(root): + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for phrase in _contains_stale_language(text): + errors.append(f"stale public wording {phrase!r} in {path.relative_to(root)}") + + examples_root = root / "examples" + if examples_root.is_dir(): + _check_example_tree(examples_root, errors) + + return errors + + +def _check_example_tree(directory: Path, errors: List[str]) -> bool: + """Validate curated leaves while allowing a readable run-name container.""" + + visible = sorted(entry for entry in directory.iterdir() if not entry.name.startswith(".")) + if not visible: + return True + + proof_path = directory / "proof.json" + readme_path = directory / "README.md" + if proof_path.exists() or readme_path.exists(): + label = directory.as_posix() + if not proof_path.is_file() or not readme_path.is_file(): + errors.append(f"example is not a standalone proof: {label}/") + return True + allowed_files = {"README.md", "proof.json"} + extra = [ + entry.name + for entry in visible + if not entry.is_dir() and entry.name not in allowed_files + ] + if extra: + errors.append( + f"example contains uncurated files: {label}/ ({', '.join(extra)})" + ) + try: + proof = json.loads(proof_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + errors.append(f"example proof is not valid JSON: {label}/ ({exc})") + return True + if not isinstance(proof, dict) or not CURATED_EXAMPLE_FIELDS.issubset(proof): + errors.append(f"example proof lacks curation fields: {label}/") + elif proof.get("curated") is not True: + errors.append(f"example is not explicitly curated: {label}/") + return True + + found_leaf = False + for entry in visible: + if entry.is_dir(): + found_leaf = _check_example_tree(entry, errors) or found_leaf + else: + errors.append( + f"example file is outside a curated proof: {entry.relative_to(directory.parent)}" + ) + if directory != directory.parents[0] and not found_leaf and visible: + errors.append(f"example container has no curated proof: {directory.as_posix()}/") + return found_leaf + + +def _check_ledger(path: Path, root: Path, errors: List[str]) -> None: + """Validate the local append-only ledger and its artifact references.""" + + parsed: List[Tuple[int, Dict[str, object]]] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as exc: + errors.append(f"ledger cannot be read: {path.relative_to(root)} ({exc})") + return + + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + errors.append(f"ledger line {line_number} is not valid JSON: {exc.msg}") + continue + if not isinstance(record, dict): + errors.append(f"ledger line {line_number} must be a JSON object") + continue + + missing = sorted(LEDGER_FIELDS.difference(record)) + if missing: + errors.append( + f"ledger line {line_number} is missing fields: {', '.join(missing)}" + ) + parsed.append((line_number, record)) + + seen: Dict[str, Dict[str, object]] = {} + recovered: Dict[str, int] = {} + for line_number, record in parsed: + run_id = record.get("run_id") + if not isinstance(run_id, str) or not run_id: + errors.append(f"ledger line {line_number} must have a non-empty run_id") + continue + duplicate_run_id = run_id in seen + if duplicate_run_id: + errors.append(f"ledger line {line_number} repeats run_id {run_id!r}") + + for timestamp_field in ("started_at", "finished_at"): + timestamp = record.get(timestamp_field) + if not isinstance(timestamp, str) or not TIMESTAMP_RE.fullmatch(timestamp): + errors.append( + f"ledger line {line_number} has an invalid {timestamp_field}" + ) + + status = record.get("status") + if status not in {"succeeded", "failed"}: + errors.append(f"ledger line {line_number} has an invalid status") + + for ref_field in ("output_ref", "proof_ref"): + ref_path = _relative_reference(root, record.get(ref_field)) + if ref_path is None or not ref_path.is_file(): + errors.append( + f"ledger line {line_number} has an invalid {ref_field}" + ) + + previous_id = record.get("previous_run_id") + relation = record.get("previous_run_relation") + if previous_id is None and relation is not None: + errors.append( + f"ledger line {line_number} has a relation without a previous run" + ) + if previous_id is not None and relation not in {"predecessor", "recovery"}: + errors.append( + f"ledger line {line_number} has an invalid previous-run relation" + ) + if previous_id is not None and not isinstance(previous_id, str): + errors.append(f"ledger line {line_number} has an invalid previous_run_id") + if isinstance(previous_id, str) and ( + previous_id == run_id or previous_id not in seen + ): + errors.append( + f"ledger line {line_number} points to a run that is not earlier in the ledger" + ) + + recovery = record.get("recovery") + if relation == "recovery" and recovery is None: + errors.append(f"ledger line {line_number} lacks recovery evidence") + if recovery is not None: + if not isinstance(recovery, dict): + errors.append(f"ledger line {line_number} has invalid recovery evidence") + continue + from_id = recovery.get("from_run_id") + if relation != "recovery": + errors.append( + f"ledger line {line_number} has recovery evidence without a recovery relation" + ) + if from_id != previous_id: + errors.append( + f"ledger line {line_number} recovery target disagrees with previous_run_id" + ) + if not isinstance(from_id, str) or seen.get(from_id, {}).get("status") != "failed": + errors.append( + f"ledger line {line_number} recovery target is not a failed run" + ) + elif from_id in recovered: + errors.append( + f"ledger line {line_number} recovers failed run {from_id!r} more than once" + ) + else: + recovered[from_id] = line_number + recovery_ref = _relative_reference(root, recovery.get("ref")) + if recovery_ref is None or not recovery_ref.is_file(): + errors.append(f"ledger line {line_number} lacks a recovery reference") + + failure = record.get("failure") + if status == "failed" and failure is None: + errors.append(f"ledger line {line_number} failed without failure evidence") + if status == "succeeded" and failure is not None: + errors.append(f"ledger line {line_number} succeeded with failure evidence") + if failure is not None: + if not isinstance(failure, dict): + errors.append(f"ledger line {line_number} has invalid failure evidence") + else: + failure_ref = _relative_reference(root, failure.get("ref")) + if failure_ref is None or not failure_ref.is_file(): + errors.append(f"ledger line {line_number} lacks a failure reference") + + if not duplicate_run_id: + seen[run_id] = record + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Check the Agentic Power BI System shell.") + parser.add_argument( + "--root", + type=Path, + default=None, + help="repository root to check (defaults to this checkout)", + ) + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = _parser().parse_args(argv) + root = (args.root or repository_root()).resolve() + errors = check_structure(root) + if errors: + print("FAIL: Agentic Power BI System checks", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print(f"PASS: structure {root}") + print(f"PASS: product {PRODUCT_NAME}") + print("PASS: visible roots workspace/ examples/ docs/") + print("PASS: stale-route and public-wording scan") + print("PASS: curated examples") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/doctor.mjs b/workspace/engine/doctor.mjs similarity index 60% rename from scripts/doctor.mjs rename to workspace/engine/doctor.mjs index f0c8073..788a01b 100644 --- a/scripts/doctor.mjs +++ b/workspace/engine/doctor.mjs @@ -4,27 +4,31 @@ import { spawnSync } from "node:child_process"; const checks = [ { cmd: "node", args: ["--version"], label: "Node.js 20+", required: true }, { cmd: "git", args: ["--version"], label: "Git", required: true }, - { cmd: "pi", args: ["--version"], label: "Pi Coding Agent", required: false }, + { cmd: "python3", args: ["--version"], label: "Python 3.9+", required: true }, { cmd: "fab", args: ["--version"], label: "Fabric CLI (fab)", required: false }, { cmd: "pbir", args: ["--version"], label: "pbir CLI", required: false }, { cmd: "pbi-tools", args: ["--version"], label: "pbi-tools", required: false }, { cmd: "pbi-tools.core", args: ["--version"], label: "pbi-tools core", required: false }, { cmd: "te", args: ["--version"], label: "Tabular Editor CLI", required: false }, { cmd: "dotnet", args: ["--version"], label: ".NET SDK/runtime", required: false }, - { cmd: "python3", args: ["--version"], label: "Python", required: false }, { cmd: "uv", args: ["--version"], label: "uv", required: false }, ]; function run(cmd, args) { - const result = spawnSync(cmd, args, { encoding: "utf8", shell: process.platform === "win32" }); + const result = spawnSync(cmd, args, { + encoding: "utf8", + shell: process.platform === "win32", + }); return { ok: result.status === 0, - text: `${result.stdout || result.stderr || ""}`.trim().split(/\r?\n/)[0] || "not found", + text: `${result.stdout || result.stderr || result.error?.message || ""}` + .trim() + .split(/\r?\n/)[0] || "not found", }; } let requiredMissing = 0; -console.log("PowerBI-template toolchain check\n"); +console.log("Agentic Power BI System toolchain check\n"); for (const check of checks) { const result = run(check.cmd, check.args); const icon = result.ok ? "✓" : check.required ? "✗" : "-"; @@ -32,11 +36,9 @@ for (const check of checks) { console.log(`${icon} ${check.label.padEnd(24)} ${result.text}`); } -console.log("\nRecommended installs:"); -console.log("- Fabric CLI: pip install ms-fabric-cli"); -console.log("- pbir CLI: uv tool install pbir-cli (or: pip install pbir-cli)"); -console.log("- Pi (optional): npm install -g @mariozechner/pi-coding-agent"); -console.log("- pbi-tools: download from https://pbi.tools/cli/"); -console.log("- Tabular Editor / DAX Studio: install desktop tools on Windows"); +console.log("\nOptional native checks are reported only when installed:"); +console.log("- pbir validate --all"); +console.log("- fab exists "); +console.log("- Tabular Editor, DAX Studio, Power BI Desktop, or pbi-tools"); process.exit(requiredMissing ? 2 : 0); diff --git a/workspace/engine/seed-guard.mjs b/workspace/engine/seed-guard.mjs new file mode 100644 index 0000000..45b1db9 --- /dev/null +++ b/workspace/engine/seed-guard.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ENGINE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(ENGINE_DIR, "../.."); +const PLACEHOLDER = ".gitkeep"; + +function relativePath(relative) { + return path.join(ROOT, relative); +} + +function checkPlaceholderDirectory(relative, errors) { + const directory = relativePath(relative); + if (!fs.existsSync(directory)) { + errors.push(`${relative}/ is missing`); + return; + } + if (!fs.lstatSync(directory).isDirectory()) { + errors.push(`${relative}/ is not a directory`); + return; + } + + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.name === PLACEHOLDER && entry.isFile()) continue; + errors.push(`${relative}/${entry.name} is mutable operational state`); + } +} + +function checkBlankLedger(errors) { + const relative = "workspace/history/runs.jsonl"; + const ledger = relativePath(relative); + if (!fs.existsSync(ledger)) { + errors.push(`${relative} is missing`); + return; + } + if (!fs.lstatSync(ledger).isFile()) { + errors.push(`${relative} is not a file`); + return; + } + if (fs.readFileSync(ledger, "utf8").trim().length > 0) { + errors.push(`${relative} contains operational run records`); + } +} + +function checkHistoryDirectory(errors) { + const relative = "workspace/history"; + const directory = relativePath(relative); + if (!fs.existsSync(directory)) { + errors.push(`${relative}/ is missing`); + return; + } + if (!fs.lstatSync(directory).isDirectory()) { + errors.push(`${relative}/ is not a directory`); + return; + } + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.name === "runs.jsonl" && entry.isFile()) continue; + errors.push(`${relative}/${entry.name} is mutable operational state`); + } +} + +function seedStateErrors() { + const errors = []; + checkBlankLedger(errors); + checkHistoryDirectory(errors); + for (const relative of [ + "workspace/briefs", + "workspace/models", + "workspace/reports", + "workspace/state", + "workspace/runs", + "workspace/learning", + ]) { + checkPlaceholderDirectory(relative, errors); + } + return errors; +} + +const errors = seedStateErrors(); +if (errors.length > 0) { + console.error( + "Agentic Power BI System seed guard: refusing to pack mutable operational state.", + ); + for (const error of errors) console.error(`- ${error}`); + console.error("Operational evidence was not changed or removed."); + process.exitCode = 1; +} diff --git a/tests/fixtures/invalid/Broken.Report/definition.pbir b/workspace/engine/tests/fixtures/invalid/Broken.Report/definition.pbir similarity index 100% rename from tests/fixtures/invalid/Broken.Report/definition.pbir rename to workspace/engine/tests/fixtures/invalid/Broken.Report/definition.pbir diff --git a/tests/fixtures/valid/Repair.Report/definition.pbir b/workspace/engine/tests/fixtures/valid/Repair.Report/definition.pbir similarity index 100% rename from tests/fixtures/valid/Repair.Report/definition.pbir rename to workspace/engine/tests/fixtures/valid/Repair.Report/definition.pbir diff --git a/tests/fixtures/valid/Repair.Report/definition/report.json b/workspace/engine/tests/fixtures/valid/Repair.Report/definition/report.json similarity index 100% rename from tests/fixtures/valid/Repair.Report/definition/report.json rename to workspace/engine/tests/fixtures/valid/Repair.Report/definition/report.json diff --git a/tests/fixtures/valid/Repair.SemanticModel/definition/model.tmdl b/workspace/engine/tests/fixtures/valid/Repair.SemanticModel/definition/model.tmdl similarity index 100% rename from tests/fixtures/valid/Repair.SemanticModel/definition/model.tmdl rename to workspace/engine/tests/fixtures/valid/Repair.SemanticModel/definition/model.tmdl diff --git a/workspace/engine/tests/test-validator.mjs b/workspace/engine/tests/test-validator.mjs new file mode 100644 index 0000000..51af9d8 --- /dev/null +++ b/workspace/engine/tests/test-validator.mjs @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../..", +); + +const cases = [ + { path: "workspace/engine/tests/fixtures/valid", expected: 0 }, + { path: "workspace/engine/tests/fixtures/invalid", expected: 2 }, +]; + +function runValidatorFixtures() { + for (const testCase of cases) { + const result = spawnSync( + process.execPath, + ["workspace/engine/validate-pbip.mjs", testCase.path], + { cwd: ROOT, encoding: "utf8" }, + ); + if (result.status !== testCase.expected) { + process.stderr.write( + `Validator case ${testCase.path} returned ${result.status}; expected ${testCase.expected}.\n${result.stdout}${result.stderr}`, + ); + process.exit(1); + } + } + process.stdout.write("PBIP validator fixtures: PASS\n"); +} + +function temporarySeed() { + const parent = fs.mkdtempSync( + path.join(os.tmpdir(), "agentic-powerbi-system-pack-"), + ); + const root = path.join(parent, "seed"); + fs.cpSync(ROOT, root, { + recursive: true, + filter(source) { + const relative = path.relative(ROOT, source); + if (!relative) return true; + const parts = relative.split(path.sep); + return !parts.includes(".git") && !parts.includes("node_modules") && !parts.includes("__pycache__"); + }, + }); + return { + root, + cleanup() { + fs.rmSync(parent, { recursive: true, force: true }); + }, + }; +} + +function pack(root) { + return spawnSync("npm", ["pack", "--dry-run", "--json"], { + cwd: root, + encoding: "utf8", + timeout: 60_000, + }); +} + +function packFiles(result) { + assert.equal( + result.status, + 0, + `npm pack failed unexpectedly:\n${result.stdout}\n${result.stderr}`, + ); + const payload = JSON.parse(result.stdout); + assert.equal(Array.isArray(payload), true); + assert.equal(Array.isArray(payload[0]?.files), true); + return payload[0].files.map((file) => file.path); +} + +function snapshotFiles(root, relative) { + const base = path.join(root, relative); + const output = []; + + function visit(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name))) { + const absolute = path.join(directory, entry.name); + const relativePath = path.relative(root, absolute).split(path.sep).join("/"); + if (entry.isDirectory()) { + visit(absolute); + } else if (entry.isSymbolicLink()) { + output.push(`${relativePath}:link:${fs.readlinkSync(absolute)}`); + } else { + const digest = crypto + .createHash("sha256") + .update(fs.readFileSync(absolute)) + .digest("hex"); + output.push(`${relativePath}:file:${digest}`); + } + } + } + + visit(base); + return output; +} + +function operationalSnapshot(root) { + const ledger = path.join(root, "workspace", "history", "runs.jsonl"); + return { + ledger: crypto + .createHash("sha256") + .update(fs.readFileSync(ledger)) + .digest("hex"), + runs: snapshotFiles(root, "workspace/runs"), + state: snapshotFiles(root, "workspace/state"), + }; +} + +function runTracer(root, args) { + const result = spawnSync( + "python3", + ["workspace/engine/tracer.py", ...args], + { cwd: root, encoding: "utf8" }, + ); + assert.equal( + result.status, + 0, + `tracer failed for ${args.join(" ")}:\n${result.stdout}\n${result.stderr}`, + ); +} + +function testCleanSeedPackage() { + const temporary = temporarySeed(); + try { + const files = packFiles(pack(temporary.root)); + assert(files.includes("workspace/engine/seed-guard.mjs")); + assert(files.includes("workspace/history/runs.jsonl")); + assert(files.includes("workspace/runs/.gitkeep")); + assert(!files.some((file) => file.startsWith("workspace/runs/run-"))); + assert(!files.some((file) => file.startsWith("workspace/state/") && !file.endsWith(".gitkeep"))); + } finally { + temporary.cleanup(); + } +} + +function testCuratedExampleRemainsPackageable() { + const temporary = temporarySeed(); + try { + const example = path.join(temporary.root, "examples", "curated-seed"); + fs.mkdirSync(example, { recursive: true }); + fs.writeFileSync(example + "/README.md", "# Curated seed proof\n"); + fs.writeFileSync( + example + "/proof.json", + JSON.stringify({ curated: true, example: "curated-seed", source_run_id: "seed", status: "succeeded" }) + "\n", + ); + + const files = packFiles(pack(temporary.root)); + assert(files.includes("examples/curated-seed/README.md")); + assert(files.includes("examples/curated-seed/proof.json")); + } finally { + temporary.cleanup(); + } +} + +function testPackRefusesTracedStateAndPreservesEvidence() { + const temporary = temporarySeed(); + try { + runTracer(temporary.root, ["--promote-example"]); + runTracer(temporary.root, ["--simulate-failure"]); + runTracer(temporary.root, ["--recover", "--promote-example"]); + const before = operationalSnapshot(temporary.root); + + const result = pack(temporary.root); + assert.notEqual(result.status, 0); + assert.match( + `${result.stdout}\n${result.stderr}`, + /seed guard: refusing to pack mutable operational state/i, + ); + assert.deepEqual(operationalSnapshot(temporary.root), before); + assert(fs.existsSync(path.join(temporary.root, "examples", "powerbi-system-proof", "run-0001", "proof.json"))); + assert(fs.existsSync(path.join(temporary.root, "workspace", "runs", "run-0003", "recovery.json"))); + } finally { + temporary.cleanup(); + } +} + +function testPackRefusesStateFileAndPreservesIt() { + const temporary = temporarySeed(); + try { + const stateFile = path.join(temporary.root, "workspace", "state", "local-state.json"); + fs.writeFileSync(stateFile, JSON.stringify({ fixture: true }) + "\n"); + const before = operationalSnapshot(temporary.root); + + const result = pack(temporary.root); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}\n${result.stderr}`, /workspace\/state\/local-state\.json/); + assert.deepEqual(operationalSnapshot(temporary.root), before); + } finally { + temporary.cleanup(); + } +} + +runValidatorFixtures(); +testCleanSeedPackage(); +testCuratedExampleRemainsPackageable(); +testPackRefusesTracedStateAndPreservesEvidence(); +testPackRefusesStateFileAndPreservesIt(); +process.stdout.write( + "Seed packaging guard: PASS (clean allow; curated allow; traced success/failure/recovery deny+preserve; state deny+preserve)\n", +); diff --git a/workspace/engine/tests/test_template.py b/workspace/engine/tests/test_template.py new file mode 100644 index 0000000..af58697 --- /dev/null +++ b/workspace/engine/tests/test_template.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from shutil import copytree, rmtree + + +ROOT = Path(__file__).resolve().parents[3] +ENGINE = ROOT / "workspace" / "engine" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load {name} from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +checks = _load_module("agentic_powerbi_checks", ENGINE / "checks.py") +tracer = _load_module("agentic_powerbi_tracer", ENGINE / "tracer.py") + + +class AgenticPowerBISystemTests(unittest.TestCase): + def test_repository_shell_is_canonical(self) -> None: + self.assertEqual(checks.check_structure(ROOT), []) + self.assertTrue((ROOT / "workspace" / "history" / "runs.jsonl").is_file()) + + def test_required_path_is_enforced(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + (root / ".agents" / "skills" / "validation" / "SKILL.md").unlink() + + errors = checks.check_structure(root) + + self.assertIn( + "required path is missing: .agents/skills/validation/SKILL.md", + errors, + ) + + def _temporary_seed(self, source_root: Path = ROOT): + temporary = tempfile.TemporaryDirectory() + temp_root = Path(temporary.name) / "seed" + copytree( + source_root, + temp_root, + ignore=lambda _path, names: {".git", "__pycache__"}.intersection(names), + ) + self._reset_operational_state(temp_root) + return temporary, temp_root + + @staticmethod + def _reset_operational_state(root: Path) -> None: + history = root / "workspace" / "history" / "runs.jsonl" + history.write_text("", encoding="utf-8") + for relative in ( + "workspace/runs", + "workspace/learning", + "workspace/state", + "examples", + ): + directory = root / relative + if not directory.is_dir(): + continue + for child in directory.iterdir(): + if child.name == ".gitkeep": + continue + if child.is_dir() and not child.is_symlink(): + rmtree(child) + else: + child.unlink() + + def test_success_route_appends_then_promotes(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + result = tracer.trace_once(root, promote_example=True) + + self.assertEqual(result.run_id, "run-0001") + self.assertEqual(result.status, "succeeded") + self.assertIsNone(result.previous_run_id) + self.assertIsNone(result.previous_run_relation) + self.assertTrue(result.output_path.is_file()) + self.assertTrue(result.proof_path.is_file()) + self.assertTrue(result.example_path is not None and result.example_path.is_dir()) + self.assertEqual(checks.check_structure(root), []) + ledger_lines = (root / "workspace/history/runs.jsonl").read_text().splitlines() + self.assertEqual(len([line for line in ledger_lines if line.strip()]), 1) + record = json.loads(ledger_lines[-1]) + self.assertEqual(record["run_id"], "run-0001") + self.assertEqual(record["input_ref"], tracer.INPUT_REF) + self.assertEqual(record["status"], "succeeded") + self.assertIsNone(record["failure"]) + self.assertIsNone(record["recovery"]) + proof = json.loads(result.proof_path.read_text()) + self.assertEqual( + proof["curated_example_ref"], + "examples/powerbi-system-proof/run-0001/", + ) + example_proof = json.loads((result.example_path / "proof.json").read_text()) + self.assertTrue(example_proof["curated"]) + self.assertEqual(example_proof["product"], tracer.PRODUCT_NAME) + + def test_second_run_inspects_predecessor_and_history_is_append_only(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + first = tracer.trace_once(root) + second = tracer.trace_once(root) + + self.assertEqual(first.run_id, "run-0001") + self.assertEqual(second.run_id, "run-0002") + self.assertEqual(second.previous_run_id, "run-0001") + self.assertEqual(second.previous_run_relation, "predecessor") + ledger_lines = [ + line + for line in (root / "workspace/history/runs.jsonl").read_text().splitlines() + if line.strip() + ] + self.assertEqual(len(ledger_lines), 2) + self.assertEqual(json.loads(ledger_lines[1])["previous_run_id"], "run-0001") + self.assertEqual( + json.loads(ledger_lines[1])["previous_run_relation"], "predecessor" + ) + + def test_failure_and_recovery_evidence_are_linked(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + failed = tracer.trace_once(root, simulate_failure=True) + continuation = tracer.trace_once(root) + recovered = tracer.trace_once(root, recover=True, promote_example=True) + + self.assertEqual(failed.status, "failed") + self.assertTrue(failed.failure_path is not None and failed.failure_path.is_file()) + self.assertEqual(continuation.run_id, "run-0002") + self.assertEqual(continuation.previous_run_id, "run-0001") + self.assertEqual(continuation.previous_run_relation, "predecessor") + self.assertEqual(recovered.status, "succeeded") + self.assertEqual(recovered.previous_run_id, "run-0001") + self.assertEqual(recovered.previous_run_relation, "recovery") + self.assertTrue( + recovered.recovery_path is not None and recovered.recovery_path.is_file() + ) + with self.assertRaises(tracer.TraceError): + tracer.trace_once(root, recover=True) + records = [ + json.loads(line) + for line in (root / "workspace/history/runs.jsonl").read_text().splitlines() + if line.strip() + ] + self.assertEqual( + records[0]["failure"]["ref"], + "workspace/runs/run-0001/failure.json", + ) + self.assertIsNone(records[0]["recovery"]) + self.assertIsNone(records[1]["recovery"]) + self.assertEqual(records[2]["recovery"]["from_run_id"], "run-0001") + self.assertEqual( + records[2]["recovery"]["ref"], + "workspace/runs/run-0003/recovery.json", + ) + self.assertEqual(records[0]["previous_run_relation"], None) + self.assertEqual(records[1]["previous_run_relation"], "predecessor") + self.assertEqual(records[2]["previous_run_relation"], "recovery") + self.assertEqual(checks.check_structure(root), []) + + def test_recovery_is_single_use_per_failed_run(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + failed = tracer.trace_once(root, simulate_failure=True) + recovered = tracer.trace_once(root, recover=True) + + self.assertEqual(failed.run_id, "run-0001") + self.assertEqual(recovered.run_id, "run-0002") + self.assertEqual(recovered.previous_run_id, "run-0001") + with self.assertRaisesRegex(TraceErrorAlias, "previous unresolved failed"): + tracer.trace_once(root, recover=True) + + second_failure = tracer.trace_once(root, simulate_failure=True) + second_recovery = tracer.trace_once(root, recover=True) + self.assertEqual(second_failure.run_id, "run-0003") + self.assertEqual(second_recovery.run_id, "run-0004") + self.assertEqual(second_recovery.previous_run_relation, "recovery") + + def test_stale_public_wording_is_rejected(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + bad_wording = "This is a " + "hand" + " off " + "sche" + "ma.\n" + (root / "docs" / "bad.md").write_text(bad_wording, encoding="utf-8") + + errors = checks.check_structure(root) + + self.assertTrue(any("stale public wording" in error for error in errors)) + + def test_legacy_root_route_is_rejected(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + (root / ("prom" + "pts")).mkdir() + + errors = checks.check_structure(root) + + self.assertTrue(any("legacy root path" in error for error in errors)) + + def test_ledger_checker_rejects_repeated_recovery_reference(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + tracer.trace_once(root, simulate_failure=True) + tracer.trace_once(root, recover=True) + records = [ + json.loads(line) + for line in (root / "workspace/history/runs.jsonl").read_text().splitlines() + if line.strip() + ] + duplicate = dict(records[1]) + duplicate["run_id"] = "run-0003" + duplicate["output_ref"] = "workspace/runs/run-0003/output.json" + duplicate["proof_ref"] = "workspace/runs/run-0003/proof.json" + history = root / "workspace/history/runs.jsonl" + history.write_text( + "\n".join(json.dumps(record, sort_keys=True) for record in records + [duplicate]) + + "\n", + encoding="utf-8", + ) + + errors = checks.check_structure(root) + + self.assertTrue(any("more than once" in error for error in errors)) + + def test_source_operational_state_is_unchanged_by_temporary_proof(self) -> None: + source_history = (ROOT / "workspace/history/runs.jsonl").read_text( + encoding="utf-8" + ) + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + tracer.trace_once(root, promote_example=True) + tracer.trace_once(root, simulate_failure=True) + tracer.trace_once(root, recover=True, promote_example=True) + + self.assertEqual( + (ROOT / "workspace/history/runs.jsonl").read_text(encoding="utf-8"), + source_history, + ) + + def test_cli_supports_explicit_root(self) -> None: + temporary, root = self._temporary_seed() + self.addCleanup(temporary.cleanup) + + result = subprocess.run( + [ + sys.executable, + str(ENGINE / "tracer.py"), + "--root", + str(root), + "--promote-example", + ], + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("system: Agentic Power BI System", result.stdout) + self.assertIn("run: run-0001", result.stdout) + + +TraceErrorAlias = tracer.TraceError + + +if __name__ == "__main__": + unittest.main() diff --git a/workspace/engine/tracer.py b/workspace/engine/tracer.py new file mode 100644 index 0000000..d2815f5 --- /dev/null +++ b/workspace/engine/tracer.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Deterministic filesystem proof tracer for the Agentic Power BI System. + +The tracer is a small local workflow, not a framework. It inspects the local +run ledger, writes one run directory, appends one ledger record, and promotes a +standalone example only when explicitly requested. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional + + +DEFAULT_TIMESTAMP = "2026-01-01T00:00:00Z" +PRODUCT_NAME = "Agentic Power BI System" +ROUTE = "system-proof" +INPUT_REF = "fixture://agentic-powerbi-system/system-proof" +DECISION_REF = "workspace/PROJECT-BRIEF.md" +RUN_ID_PATTERN = re.compile(r"^run-(\d{4,})$") +SPECIALIST_ROUTES = ( + "powerbi", + "pbip", + "dax", + "report", + "fabric", + "validation", +) + + +class TraceError(RuntimeError): + """Raised when the local trace cannot preserve its evidence.""" + + +@dataclass(frozen=True) +class TraceResult: + run_id: str + status: str + output_path: Path + proof_path: Path + ledger_path: Path + example_path: Optional[Path] + previous_run_id: Optional[str] + previous_run_relation: Optional[str] + inspected_prior_runs: int + failure_path: Optional[Path] + recovery_path: Optional[Path] + + +def repository_root() -> Path: + """Return the repository root derived from this file, not the cwd.""" + + return Path(__file__).resolve().parents[2] + + +def _relative(path: Path, root: Path) -> str: + return path.relative_to(root).as_posix() + + +def _write_json(path: Path, value: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _read_ledger(ledger_path: Path) -> List[Dict[str, Any]]: + if not ledger_path.exists(): + return [] + + records: List[Dict[str, Any]] = [] + for line_number, line in enumerate( + ledger_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise TraceError( + f"ledger line {line_number} is not valid JSON: {exc.msg}" + ) from exc + if not isinstance(record, dict): + raise TraceError(f"ledger line {line_number} must be an object") + records.append(record) + return records + + +def _next_run_id(records: List[Dict[str, Any]]) -> str: + numbers = [] + for record in records: + match = RUN_ID_PATTERN.match(str(record.get("run_id", ""))) + if match: + numbers.append(int(match.group(1))) + return f"run-{(max(numbers, default=0) + 1):04d}" + + +def _append_ledger(ledger_path: Path, record: Dict[str, Any]) -> None: + ledger_path.parent.mkdir(parents=True, exist_ok=True) + with ledger_path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record, sort_keys=True) + "\n") + + +def _promote_example( + root: Path, + run_id: str, + output: Dict[str, Any], + proof: Dict[str, Any], +) -> Path: + """Write a self-contained proof only after explicit curation.""" + + example_dir = root / "examples" / "powerbi-system-proof" / run_id + example_proof = { + "assertions": proof["assertions"], + "curated": True, + "example": "powerbi-system-proof", + "product": PRODUCT_NAME, + "result": output["result"], + "route": output["route"], + "source_proof_ref": proof["proof_ref"], + "source_run_id": run_id, + "specialist_routes": output["specialist_routes"], + "status": output["status"], + } + _write_json(example_dir / "proof.json", example_proof) + (example_dir / "README.md").write_text( + "# Curated Agentic Power BI System proof\n\n" + "This directory is an intentionally curated, standalone proof of one " + "deterministic System route. It contains fixture evidence only; it " + "does not claim a live Power BI model or service result.\n\n" + f"- Product: `{PRODUCT_NAME}`\n" + f"- Run: `{run_id}`\n" + f"- Route: `{output['route']}`\n" + f"- Status: `{output['status']}`\n" + f"- Result: {output['result']}\n\n" + "The complete assertions are in `proof.json`. This example is a " + "readable proof artifact; it is not an executable dependency.\n", + encoding="utf-8", + ) + return example_dir + + +def trace_once( + root: Path, + *, + promote_example: bool = False, + simulate_failure: bool = False, + recover: bool = False, + timestamp: str = DEFAULT_TIMESTAMP, +) -> TraceResult: + """Run one deterministic route and preserve its evidence. + + ``recover`` consumes the latest relevant failed run. The explicit root + keeps the workflow testable in a fresh temporary copy. + """ + + root = root.resolve() + if not root.is_dir(): + raise TraceError(f"repository root is not a directory: {root}") + if simulate_failure and recover: + raise TraceError("choose either --simulate-failure or --recover") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", timestamp): + raise TraceError("timestamp must use UTC form YYYY-MM-DDTHH:MM:SSZ") + + ledger_path = root / "workspace" / "history" / "runs.jsonl" + records = _read_ledger(ledger_path) + relevant = [ + record for record in records if record.get("input_ref") == INPUT_REF + ] + recovered_failures = set() + for record in relevant: + recovery = record.get("recovery") + if isinstance(recovery, dict) and recovery.get("from_run_id"): + recovered_failures.add(recovery.get("from_run_id")) + failed_prior = next( + ( + record + for record in reversed(relevant) + if record.get("status") == "failed" + and record.get("run_id") not in recovered_failures + ), + None, + ) + if recover and failed_prior is None: + raise TraceError("--recover requires a previous unresolved failed System run") + + if recover: + previous_run_id = failed_prior.get("run_id") + else: + previous = relevant[-1] if relevant else None + previous_run_id = previous.get("run_id") if previous else None + + if previous_run_id is None: + previous_run_relation = None + elif recover: + previous_run_relation = "recovery" + else: + previous_run_relation = "predecessor" + + run_id = _next_run_id(records) + run_dir = root / "workspace" / "runs" / run_id + if run_dir.exists(): + raise TraceError(f"run directory already exists: {_relative(run_dir, root)}") + run_dir.mkdir(parents=True) + + output_path = run_dir / "output.json" + proof_path = run_dir / "proof.json" + failure_path: Optional[Path] = None + recovery_path: Optional[Path] = None + + input_record = { + "decision_ref": DECISION_REF, + "input_ref": INPUT_REF, + "request_kind": "deterministic-system-proof", + "route": ROUTE, + "specialist_routes": list(SPECIALIST_ROUTES), + "system": PRODUCT_NAME, + } + _write_json(run_dir / "input.json", input_record) + + if simulate_failure: + status = "failed" + output = { + "audience": "Power BI model and report owners", + "decision": "Confirm that the decision-first route preserves its evidence.", + "previous_run_id": previous_run_id, + "previous_run_relation": previous_run_relation, + "result": "The deterministic System route stopped at its recoverable fixture.", + "route": ROUTE, + "run_id": run_id, + "specialist_routes": list(SPECIALIST_ROUTES), + "status": status, + "system": PRODUCT_NAME, + } + failure_path = run_dir / "failure.json" + failure = { + "code": "SYSTEM_PROOF_FAILURE", + "message": "The deterministic recoverable failure fixture was requested.", + "recoverable": True, + "run_id": run_id, + } + _write_json(failure_path, failure) + recovery: Optional[Dict[str, Any]] = None + assertions = [ + "the primary System route selected the deterministic fixture", + "the run recorded a deterministic recoverable failure artifact", + "the failed run remained available for a later recovery", + ] + else: + status = "succeeded" + recovered_from = failed_prior.get("run_id") if recover and failed_prior else None + output = { + "audience": "Power BI model and report owners", + "decision": "Confirm that the decision-first route preserves its evidence.", + "previous_run_id": previous_run_id, + "previous_run_relation": previous_run_relation, + "recovered_from": recovered_from, + "result": "The Agentic Power BI System route completed deterministically without live-service claims.", + "route": ROUTE, + "run_id": run_id, + "specialist_routes": list(SPECIALIST_ROUTES), + "status": status, + "system": PRODUCT_NAME, + } + recovery = None + if recover: + recovery_path = run_dir / "recovery.json" + recovery = { + "action": "reroute the System proof after its recorded failure", + "from_run_id": failed_prior.get("run_id"), + "run_id": run_id, + "status": "recovered", + } + _write_json(recovery_path, recovery) + assertions = [ + "the primary System route inspected a prior failed run", + "the route completed after the recorded failure", + "recovery evidence points to the failed predecessor", + ] + else: + assertions = [ + "the primary System route inspected the local run ledger", + "Power BI specialist route references were recorded", + "output and proof were written under workspace/runs", + "the append-only ledger received one run record", + ] + + _write_json(output_path, output) + proof = { + "assertions": assertions, + "curated_example_ref": ( + f"examples/powerbi-system-proof/{run_id}/" if promote_example else None + ), + "decision_ref": DECISION_REF, + "failure_ref": _relative(failure_path, root) if failure_path else None, + "input_ref": INPUT_REF, + "ledger_ref": f"workspace/history/runs.jsonl#{run_id}", + "output_ref": _relative(output_path, root), + "previous_run_id": previous_run_id, + "previous_run_relation": previous_run_relation, + "proof_ref": _relative(proof_path, root), + "recovery_ref": _relative(recovery_path, root) if recovery_path else None, + "route": ROUTE, + "run_id": run_id, + "specialist_routes": list(SPECIALIST_ROUTES), + "status": status, + "system": PRODUCT_NAME, + } + _write_json(proof_path, proof) + + ledger_record = { + "failure": ( + { + "code": "SYSTEM_PROOF_FAILURE", + "ref": _relative(failure_path, root), + } + if failure_path + else None + ), + "finished_at": timestamp, + "input_ref": INPUT_REF, + "output_ref": _relative(output_path, root), + "previous_run_id": previous_run_id, + "previous_run_relation": previous_run_relation, + "proof_ref": _relative(proof_path, root), + "recovery": ( + { + "from_run_id": failed_prior.get("run_id"), + "ref": _relative(recovery_path, root), + } + if recovery_path and failed_prior + else None + ), + "route": ROUTE, + "run_id": run_id, + "specialist_routes": list(SPECIALIST_ROUTES), + "started_at": timestamp, + "status": status, + "system": PRODUCT_NAME, + } + _append_ledger(ledger_path, ledger_record) + + example_path = None + if promote_example: + example_path = _promote_example(root, run_id, output, proof) + + return TraceResult( + run_id=run_id, + status=status, + output_path=output_path, + proof_path=proof_path, + ledger_path=ledger_path, + example_path=example_path, + previous_run_id=previous_run_id, + previous_run_relation=previous_run_relation, + inspected_prior_runs=len(relevant), + failure_path=failure_path, + recovery_path=recovery_path, + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run the deterministic Agentic Power BI System proof." + ) + parser.add_argument( + "--promote-example", + action="store_true", + help="intentionally curate this run into examples/", + ) + parser.add_argument( + "--simulate-failure", + action="store_true", + help="record the deterministic recoverable failure path", + ) + parser.add_argument( + "--recover", + action="store_true", + help="recover the latest failed System proof run", + ) + parser.add_argument( + "--root", + type=Path, + default=None, + help="repository root to trace (defaults to this checkout)", + ) + parser.add_argument( + "--timestamp", + default=DEFAULT_TIMESTAMP, + help=f"UTC timestamp (default: {DEFAULT_TIMESTAMP})", + ) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + args = _parser().parse_args(argv) + root = (args.root or repository_root()).resolve() + try: + result = trace_once( + root, + promote_example=args.promote_example, + simulate_failure=args.simulate_failure, + recover=args.recover, + timestamp=args.timestamp, + ) + except TraceError as exc: + print(f"trace failed: {exc}", file=sys.stderr) + return 1 + + print(f"system: {PRODUCT_NAME}") + print(f"route: {ROUTE}") + print(f"inspected_prior_runs: {result.inspected_prior_runs}") + print(f"run: {result.run_id}") + print(f"status: {result.status}") + print(f"output: {_relative(result.output_path, root)}") + print(f"proof: {_relative(result.proof_path, root)}") + print(f"ledger: {_relative(result.ledger_path, root)}") + if result.failure_path: + print(f"failure: {_relative(result.failure_path, root)}") + if result.recovery_path: + print(f"recovery: {_relative(result.recovery_path, root)}") + if result.example_path: + print(f"curated_example: {_relative(result.example_path, root)}/") + else: + print("curated_example: none") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate-pbip.mjs b/workspace/engine/validate-pbip.mjs similarity index 98% rename from scripts/validate-pbip.mjs rename to workspace/engine/validate-pbip.mjs index 9334524..189d944 100644 --- a/scripts/validate-pbip.mjs +++ b/workspace/engine/validate-pbip.mjs @@ -34,7 +34,7 @@ function walk(dir, predicate, out = []) { if (!isDir(dir)) return out; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, entry.name); - if (entry.name === ".git" || entry.name === "node_modules" || entry.name === ".pi") continue; + if (entry.name === ".git" || entry.name === "node_modules") continue; if (ignoreTests && entry.name === "tests") continue; if (entry.isDirectory()) walk(full, predicate, out); else if (predicate(full)) out.push(full); @@ -49,7 +49,7 @@ function findDirs(start, suffix) { if (path.basename(dir).endsWith(suffix)) out.push(dir); for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { if (!entry.isDirectory()) continue; - if ([".git", "node_modules", ".pi"].includes(entry.name)) continue; + if ([".git", "node_modules"].includes(entry.name)) continue; if (ignoreTests && entry.name === "tests") continue; visit(path.join(dir, entry.name)); } diff --git a/workspace/history/runs.jsonl b/workspace/history/runs.jsonl new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/history/runs.jsonl @@ -0,0 +1 @@ + diff --git a/workspace/learning/.gitkeep b/workspace/learning/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/learning/.gitkeep @@ -0,0 +1 @@ + diff --git a/workspace/models/.gitkeep b/workspace/models/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/models/.gitkeep @@ -0,0 +1 @@ + diff --git a/workspace/reports/.gitkeep b/workspace/reports/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/reports/.gitkeep @@ -0,0 +1 @@ + diff --git a/workspace/runs/.gitkeep b/workspace/runs/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/runs/.gitkeep @@ -0,0 +1 @@ + diff --git a/workspace/state/.gitkeep b/workspace/state/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/workspace/state/.gitkeep @@ -0,0 +1 @@ +