From 6ee3372560275c3b53aa93a1dbbe74b7c11274e3 Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 20:50:33 -0400 Subject: [PATCH 1/7] feat: add test-pr skill for testing starflow PRs across downstream repos Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/SKILL.md | 91 ++++ .github/skills/test-pr/test-downstream.py | 582 ++++++++++++++++++++++ 2 files changed, 673 insertions(+) create mode 100644 .github/skills/test-pr/SKILL.md create mode 100755 .github/skills/test-pr/test-downstream.py diff --git a/.github/skills/test-pr/SKILL.md b/.github/skills/test-pr/SKILL.md new file mode 100644 index 0000000..58aa0af --- /dev/null +++ b/.github/skills/test-pr/SKILL.md @@ -0,0 +1,91 @@ +--- +name: test-pr +description: Test a starflow PR against all downstream team repositories by creating draft PRs pointing shared workflows to the PR branch and linking to test runs. +--- + +# Test PR (Downstream Integration Testing) + +This skill helps developers test changes made in a `canonical/starflow` pull request against all team repositories that consume starflow's shared workflows and actions. + +It automates: + +1. Detecting the current starflow PR number, target branch/ref, and repository owner. +2. Finding all team repositories in `canonical` that use starflow workflows or actions. +3. For each downstream repository: + - Creating a dedicated test branch `test/starflow-pr-`. + - Updating all `.github/workflows/*.yaml` references to point to the starflow PR branch. + - Pushing the branch and opening a Draft PR titled `[DO NOT MERGE] Test starflow PR `. +4. Listing the resulting Draft PRs along with direct links to the triggered GitHub Actions workflow runs. +5. Providing cleanup capabilities once downstream testing is complete. + +## When to Use + +Invoke this skill when: + +- The user asks to "test this PR", "test PR #123", or "test against downstream repositories". +- The user wants to verify whether a change to starflow's shared workflows/actions breaks any consuming projects (e.g. Snapcraft, Charmcraft, Rockcraft, Craft CLI, Craft Application, Craft Parts, Craft Providers, etc.). +- The user requests draft PRs to test workflow runs in team repositories. +- The user wants to clean up / close test PRs and branches created during previous test runs. + +## Prerequisites + +- `gh` CLI authenticated with repo write permissions (`gh auth status`). +- `git` and `python3` available in the environment. + +## Usage & Execution + +The skill includes a dedicated helper script located at `.github/skills/test-pr/test-downstream.py`. + +### 1. Identify PR and Target Branch + +Identify the starflow PR number and branch to test: + +- If provided in the user prompt (e.g., "test PR 42" or "test PR on branch feature-xyz"), use that. +- Otherwise, run `gh pr view --json number,headRefName,headRepositoryOwner` or inspect `git branch --show-current`. + +### 2. Execute Downstream Tester + +Run the helper script from the repository root: + +```bash +# Test across all downstream repositories +python3 .github/skills/test-pr/test-downstream.py --pr + +# Test with an explicit branch/ref or fork owner +python3 .github/skills/test-pr/test-downstream.py --pr --branch --owner + +# Test specific repositories only +python3 .github/skills/test-pr/test-downstream.py --pr --repos canonical/snapcraft canonical/charmcraft + +# Dry run (verify workflow substitutions without pushing or creating PRs) +python3 .github/skills/test-pr/test-downstream.py --pr --dry-run +``` + +### 3. Present Results to the User + +Present the results clearly in a table and list, including: + +- Repository name +- Draft PR link (titled `[DO NOT MERGE] Test starflow PR ###`) +- Direct links to triggered GitHub Actions workflow runs +- Actions overview URL for each repository + +Example output format: + +```markdown +### Downstream Test PRs for Starflow PR #42 + +| Repository | Draft PR | Test Workflow Runs | +| :----------------------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **canonical/snapcraft** | [#123](https://github.com/canonical/snapcraft/pull/123) | [QA](https://github.com/canonical/snapcraft/actions/runs/1001) (in_progress)
[TICS](https://github.com/canonical/snapcraft/actions/runs/1002) (in_progress) | +| **canonical/charmcraft** | [#456](https://github.com/canonical/charmcraft/pull/456) | [QA](https://github.com/canonical/charmcraft/actions/runs/1003) (in_progress) | +| **canonical/rockcraft** | [#789](https://github.com/canonical/rockcraft/pull/789) | [QA](https://github.com/canonical/rockcraft/actions/runs/1004) (in_progress) | +``` + +### 4. Cleanup + +When testing is finished or when requested by the user, close the draft PRs and delete the test branches: + +```bash +python3 .github/skills/test-pr/test-downstream.py --pr --cleanup +``` diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py new file mode 100755 index 0000000..4afb4fb --- /dev/null +++ b/.github/skills/test-pr/test-downstream.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 +""" +Test Starflow PRs against downstream repositories. + +This script creates draft PRs in downstream team repositories pointing their +shared GitHub Actions workflows to a specific branch/PR of starflow, and lists +the resulting test workflow runs. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, List, Optional, Tuple + +DEFAULT_TEAM_REPOS = [ + "canonical/snapcraft", + "canonical/charmcraft", + "canonical/rockcraft", + "canonical/craft-application", + "canonical/craft-cli", + "canonical/craft-parts", + "canonical/craft-providers", + "canonical/craft-store", + "canonical/craft-grammar", + "canonical/craft-artifacts", + "canonical/starbase", + "canonical/debcraft", + "canonical/sdkcraft", + "canonical/imagecraft", + "canonical/ulwazi", + "canonical/sourcecraft", + "canonical/bld", +] + + +def run_cmd( + cmd: List[str], + cwd: Optional[Path] = None, + check: bool = True, + capture_output: bool = True, + text: bool = True, + env: Optional[dict] = None, +) -> subprocess.CompletedProcess: + """Run a command safely and return CompletedProcess.""" + full_env = os.environ.copy() + if env: + full_env.update(env) + return subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + check=check, + capture_output=capture_output, + text=text, + env=full_env, + ) + + +def get_current_starflow_pr_or_branch() -> Tuple[Optional[str], str, str]: + """ + Detect starflow PR number, target ref/branch, and owner. + Returns (pr_number, branch_ref, owner). + """ + # Try getting PR info from gh pr view + try: + proc = run_cmd(["gh", "pr", "view", "--json", "number,headRefName,headRepositoryOwner"], check=False) + if proc.returncode == 0 and proc.stdout.strip(): + data = json.loads(proc.stdout) + pr_num = str(data.get("number")) + branch = data.get("headRefName") + owner = data.get("headRepositoryOwner", {}).get("login", "canonical") + if pr_num and branch: + return pr_num, branch, owner + except Exception: + pass + + # Fallback to current git branch + try: + proc = run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"], check=False) + if proc.returncode == 0: + branch = proc.stdout.strip() + # If branch is like work/123-fix or pr-123 or similar, attempt to extract number + num_match = re.search(r"(?:pr[-/]?|#)(\d+)", branch, re.IGNORECASE) + pr_num = num_match.group(1) if num_match else None + return pr_num, branch, "canonical" + except Exception: + pass + + return None, "main", "canonical" + + +def discover_downstream_repos() -> List[str]: + """Find repositories in canonical org that use canonical/starflow in workflows.""" + repos = set(DEFAULT_TEAM_REPOS) + try: + proc = run_cmd( + [ + "gh", + "api", + "--paginate", + "search/code?q=canonical/starflow+org:canonical+path:.github/workflows", + ], + check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + # gh api with --paginate may output multiple JSON documents or one + for line in proc.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + for item in data.get("items", []): + repo_full = item.get("repository", {}).get("full_name") + if repo_full and repo_full != "canonical/starflow": + repos.add(repo_full) + except json.JSONDecodeError: + continue + except Exception: + pass + + return sorted(repos) + + +def update_workflows_in_dir(workflow_dir: Path, target_owner: str, target_ref: str) -> List[str]: + """ + Update workflow files in workflow_dir to point to target_owner/starflow at target_ref. + Returns list of modified file names. + """ + modified_files = [] + # Pattern to match: uses: /starflow(/path)?@ + # Captures: 1=quote if any, 2=subpath if any, 3=old_ref + pattern = re.compile( + r'(uses:\s*[\'"]?)[a-zA-Z0-9_.-]+/starflow(/[^@\s\r\n\'"]*)?@([^\s\r\n\'"]+)([\'"]?)' + ) + + for yaml_path in list(workflow_dir.glob("*.yaml")) + list(workflow_dir.glob("*.yml")): + try: + content = yaml_path.read_text(encoding="utf-8") + except Exception: + continue + + def repl(match: re.Match) -> str: + prefix = match.group(1) + subpath = match.group(2) or "" + suffix = match.group(4) or "" + return f"{prefix}{target_owner}/starflow{subpath}@{target_ref}{suffix}" + + new_content, count = pattern.subn(repl, content) + if count > 0 and new_content != content: + yaml_path.write_text(new_content, encoding="utf-8") + modified_files.append(yaml_path.name) + + return modified_files + + +@dataclass +class RepoResult: + repo: str + status: str + modified_files: List[str] + pr_url: Optional[str] = None + pr_number: Optional[int] = None + branch: Optional[str] = None + workflow_runs: List[dict] = None + error: Optional[str] = None + + +def process_repo( + repo: str, + pr_number: str, + target_owner: str, + target_ref: str, + branch_name: str, + dry_run: bool = False, +) -> RepoResult: + """Clone downstream repo, update workflows, push branch and create/update Draft PR.""" + repo_name = repo.split("/")[-1] + title = f"[DO NOT MERGE] Test starflow PR {pr_number}" + body = ( + f"This is an automated draft PR to test changes from starflow PR #{pr_number}.\n\n" + f"- **Starflow PR**: https://github.com/canonical/starflow/pull/{pr_number}\n" + f"- **Target Ref**: `{target_owner}/starflow@{target_ref}`\n\n" + f"⚠️ **DO NOT MERGE**: This PR is intended solely to verify downstream CI workflows." + ) + + with tempfile.TemporaryDirectory(prefix=f"starflow-test-{repo_name}-") as tmpdir: + tmp_path = Path(tmpdir) + try: + # Shallow clone default branch + clone_proc = run_cmd( + ["gh", "repo", "clone", repo, str(tmp_path), "--", "--depth=1"], + check=False, + ) + if clone_proc.returncode != 0: + return RepoResult( + repo=repo, + status="failed", + modified_files=[], + error=f"Failed to clone: {clone_proc.stderr.strip()}", + ) + + workflow_dir = tmp_path / ".github" / "workflows" + if not workflow_dir.is_dir(): + return RepoResult( + repo=repo, + status="skipped", + modified_files=[], + error="No .github/workflows directory found", + ) + + modified = update_workflows_in_dir(workflow_dir, target_owner, target_ref) + if not modified: + return RepoResult( + repo=repo, + status="skipped", + modified_files=[], + error="No starflow workflow references found to update", + ) + + if dry_run: + return RepoResult( + repo=repo, + status="dry-run", + modified_files=modified, + branch=branch_name, + ) + + # Create branch, commit, push + run_cmd(["git", "checkout", "-b", branch_name], cwd=tmp_path) + for m in modified: + run_cmd(["git", "add", f".github/workflows/{m}"], cwd=tmp_path) + + commit_msg = ( + f"ci: test against starflow PR #{pr_number} ({target_ref})\n\n" + f"[DO NOT MERGE] Point starflow workflows to {target_owner}/starflow@{target_ref}" + ) + run_cmd(["git", "commit", "-m", commit_msg], cwd=tmp_path) + + # Push branch + push_proc = run_cmd(["git", "push", "-u", "origin", branch_name, "--force"], cwd=tmp_path, check=False) + if push_proc.returncode != 0: + return RepoResult( + repo=repo, + status="failed", + modified_files=modified, + error=f"Failed to push branch: {push_proc.stderr.strip()}", + ) + + # Check if PR already exists for this branch + existing_pr = None + pr_view = run_cmd( + ["gh", "pr", "view", branch_name, "--repo", repo, "--json", "url,number"], + check=False, + ) + if pr_view.returncode == 0 and pr_view.stdout.strip(): + try: + pr_data = json.loads(pr_view.stdout) + existing_pr = (pr_data.get("url"), pr_data.get("number")) + except Exception: + pass + + pr_url = None + pr_num = None + if existing_pr: + pr_url, pr_num = existing_pr + else: + # Create draft PR + create_pr = run_cmd( + [ + "gh", + "pr", + "create", + "--repo", + repo, + "--draft", + "--title", + title, + "--body", + body, + "--head", + branch_name, + ], + cwd=tmp_path, + check=False, + ) + if create_pr.returncode == 0 and create_pr.stdout.strip(): + pr_url = create_pr.stdout.strip().splitlines()[-1] + # Parse PR number from URL + match = re.search(r"/pull/(\d+)", pr_url) + if match: + pr_num = int(match.group(1)) + else: + return RepoResult( + repo=repo, + status="failed", + modified_files=modified, + error=f"Failed to create draft PR: {create_pr.stderr.strip()}", + ) + + # Fetch workflow runs + workflow_runs = fetch_workflow_runs(repo, branch_name) + + return RepoResult( + repo=repo, + status="success", + modified_files=modified, + pr_url=pr_url, + pr_number=pr_num, + branch=branch_name, + workflow_runs=workflow_runs, + ) + + except Exception as e: + return RepoResult( + repo=repo, + status="failed", + modified_files=[], + error=str(e), + ) + + +def fetch_workflow_runs(repo: str, branch: str, retries: int = 3, delay: float = 3.0) -> List[dict]: + """Fetch recent workflow runs for a branch in a repo.""" + runs = [] + for attempt in range(retries): + if attempt > 0: + time.sleep(delay) + proc = run_cmd( + [ + "gh", + "run", + "list", + "--repo", + repo, + "--branch", + branch, + "--limit", + "10", + "--json", + "databaseId,name,url,status,conclusion,event", + ], + check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + try: + data = json.loads(proc.stdout) + if data: + runs = data + break + except Exception: + pass + return runs + + +def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str) -> None: + """Close test draft PRs and delete branches across repos.""" + print(f"Cleaning up test PRs and branches for starflow PR #{pr_number}...") + for repo in repos: + print(f"\n[{repo}]") + # Check for open PR on branch + proc = run_cmd( + ["gh", "pr", "view", branch_name, "--repo", repo, "--json", "number,url,state"], + check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + try: + data = json.loads(proc.stdout) + pr_num = data.get("number") + state = data.get("state") + if pr_num and state == "OPEN": + print(f" Closing draft PR #{pr_num}...") + run_cmd(["gh", "pr", "close", str(pr_num), "--repo", repo, "--delete-branch"], check=False) + print(f" Closed #{pr_num} and requested branch deletion.") + continue + except Exception: + pass + + # Try deleting branch directly if PR didn't delete it + del_proc = run_cmd( + ["gh", "api", "-X", "DELETE", f"repos/{repo}/git/refs/heads/{branch_name}"], + check=False, + ) + if del_proc.returncode == 0: + print(f" Deleted branch {branch_name}.") + else: + print(" No branch or PR found to clean up.") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Test starflow PRs across downstream repositories by creating draft PRs." + ) + parser.add_argument( + "--pr", + "-p", + help="Starflow PR number (e.g. 42 or #42) or URL. If omitted, detected automatically.", + ) + parser.add_argument( + "--branch", + "-b", + help="Starflow branch or ref to test. If omitted, detected from PR or current git branch.", + ) + parser.add_argument( + "--owner", + "-o", + default="canonical", + help="Repository owner/fork hosting the starflow branch (default: canonical).", + ) + parser.add_argument( + "--repos", + "-r", + nargs="+", + help="Specific downstream repos to test (e.g. canonical/snapcraft). Defaults to all team repos using starflow.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Simulate workflow changes without pushing branches or creating PRs.", + ) + parser.add_argument( + "--cleanup", + action="store_true", + help="Close draft PRs and delete test branches created for the specified PR.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Output raw JSON results.", + ) + + args = parser.parse_args() + + # Determine PR number, branch, owner + detected_pr, detected_branch, detected_owner = get_current_starflow_pr_or_branch() + + pr_num = args.pr or detected_pr + if pr_num: + # Normalize "#123" or URL to just number string + match = re.search(r"(\d+)", str(pr_num)) + pr_num = match.group(1) if match else str(pr_num) + else: + print("Error: Could not determine starflow PR number. Please pass --pr .", file=sys.stderr) + return 1 + + target_ref = args.branch or detected_branch + target_owner = args.owner or detected_owner + branch_name = f"test/starflow-pr-{pr_num}" + + # Determine target repos + if args.repos: + repos = [] + for r in args.repos: + for item in r.split(","): + item = item.strip() + if item: + if "/" not in item: + item = f"canonical/{item}" + repos.append(item) + else: + print("Discovering downstream repositories using starflow...") + repos = discover_downstream_repos() + + if args.cleanup: + cleanup_test_prs(repos, pr_num, branch_name) + return 0 + + print(f"==================================================") + print(f"Starflow PR Downstream Testing") + print(f" PR Number : #{pr_num}") + print(f" Target Ref : {target_owner}/starflow@{target_ref}") + print(f" Test Branch : {branch_name}") + print(f" PR Title : [DO NOT MERGE] Test starflow PR {pr_num}") + print(f" Repositories: {len(repos)} candidate repo(s)") + if args.dry_run: + print(f" Mode : DRY RUN (no git push or PR creation)") + print(f"==================================================\n") + + results: List[RepoResult] = [] + + for idx, repo in enumerate(repos, 1): + print(f"[{idx}/{len(repos)}] Processing {repo}...") + res = process_repo( + repo=repo, + pr_number=pr_num, + target_owner=target_owner, + target_ref=target_ref, + branch_name=branch_name, + dry_run=args.dry_run, + ) + results.append(res) + if res.status == "success": + print(f" ✓ Draft PR created: {res.pr_url}") + print(f" Modified workflows: {', '.join(res.modified_files)}") + elif res.status == "dry-run": + print(f" ✓ [Dry Run] Would modify: {', '.join(res.modified_files)}") + elif res.status == "skipped": + print(f" - Skipped: {res.error}") + else: + print(f" ✗ Failed: {res.error}") + + if args.json: + out_data = [ + { + "repo": r.repo, + "status": r.status, + "modified_files": r.modified_files, + "pr_url": r.pr_url, + "pr_number": r.pr_number, + "branch": r.branch, + "workflow_runs": r.workflow_runs or [], + "error": r.error, + } + for r in results + ] + print(json.dumps(out_data, indent=2)) + return 0 + + # Print Markdown Summary + successful = [r for r in results if r.status == "success"] + failed = [r for r in results if r.status == "failed"] + skipped = [r for r in results if r.status == "skipped"] + + print("\n" + "=" * 50) + print("## Summary of Downstream PRs & Test Workflow Runs") + print("=" * 50 + "\n") + + if successful: + print("| Repository | Draft PR | Workflow Runs / Actions |") + print("| :--- | :--- | :--- |") + for r in successful: + runs_links = [] + if r.workflow_runs: + for run in r.workflow_runs: + name = run.get("name", "Workflow") + url = run.get("url", "") + status = run.get("status", "") + runs_links.append(f"[{name}]({url}) ({status})") + if not runs_links: + actions_url = f"https://github.com/{r.repo}/actions?query=branch%3A{branch_name}" + runs_links.append(f"[All Action Runs]({actions_url})") + + runs_str = "
".join(runs_links) + print(f"| **{r.repo}** | [{r.repo}#{r.pr_number}]({r.pr_url}) | {runs_str} |") + + print("\n### Direct Links:") + for r in successful: + print(f"\n#### {r.repo}") + print(f"- **Draft PR**: {r.pr_url}") + if r.workflow_runs: + for run in r.workflow_runs: + name = run.get("name", "Workflow") + url = run.get("url", "") + status = run.get("status", "") + print(f" - [{name}] {url} ({status})") + else: + actions_url = f"https://github.com/{r.repo}/actions?query=branch%3A{branch_name}" + print(f" - Actions overview: {actions_url}") + elif args.dry_run: + print(f"Dry run complete. {len([r for r in results if r.status == 'dry-run'])} repository(ies) would be updated.") + else: + print("No draft PRs were created.") + + if failed: + print("\n### Errors / Failures:") + for r in failed: + print(f"- **{r.repo}**: {r.error}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From bd11a75414264d1cd28823922f34faf21477ab97 Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 20:54:35 -0400 Subject: [PATCH 2/7] fix: use work/bot/starflow-test-pr-### branch template and filter team repos Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/SKILL.md | 2 +- .github/skills/test-pr/test-downstream.py | 37 ++++++++++++----------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/skills/test-pr/SKILL.md b/.github/skills/test-pr/SKILL.md index 58aa0af..daf4faf 100644 --- a/.github/skills/test-pr/SKILL.md +++ b/.github/skills/test-pr/SKILL.md @@ -12,7 +12,7 @@ It automates: 1. Detecting the current starflow PR number, target branch/ref, and repository owner. 2. Finding all team repositories in `canonical` that use starflow workflows or actions. 3. For each downstream repository: - - Creating a dedicated test branch `test/starflow-pr-`. + - Creating a dedicated test branch `work/bot/starflow-test-pr-`. - Updating all `.github/workflows/*.yaml` references to point to the starflow PR branch. - Pushing the branch and opening a Draft PR titled `[DO NOT MERGE] Test starflow PR `. 4. Listing the resulting Draft PRs along with direct links to the triggered GitHub Actions workflow runs. diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 4afb4fb..70f02b2 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -99,35 +99,38 @@ def get_current_starflow_pr_or_branch() -> Tuple[Optional[str], str, str]: def discover_downstream_repos() -> List[str]: - """Find repositories in canonical org that use canonical/starflow in workflows.""" - repos = set(DEFAULT_TEAM_REPOS) + """Find team repositories in canonical org that use starflow in workflows.""" + repos = set() + # 1. Query starcraft team repos try: proc = run_cmd( - [ - "gh", - "api", - "--paginate", - "search/code?q=canonical/starflow+org:canonical+path:.github/workflows", - ], + ["gh", "api", "orgs/canonical/teams/starcraft/repos", "--paginate"], check=False, ) if proc.returncode == 0 and proc.stdout.strip(): - # gh api with --paginate may output multiple JSON documents or one for line in proc.stdout.splitlines(): - line = line.strip() - if not line: + if not line.strip(): continue try: data = json.loads(line) - for item in data.get("items", []): - repo_full = item.get("repository", {}).get("full_name") - if repo_full and repo_full != "canonical/starflow": - repos.add(repo_full) - except json.JSONDecodeError: + items = data if isinstance(data, list) else [data] + for item in items: + if isinstance(item, dict): + full_name = item.get("full_name") + archived = item.get("archived", False) + private = item.get("private", False) + # Exclude starflow itself, private repos, or archived repos + if full_name and full_name != "canonical/starflow" and not archived and not private: + repos.add(full_name) + except Exception: continue except Exception: pass + # 2. Add defaults if team query was empty + if not repos: + repos.update(DEFAULT_TEAM_REPOS) + return sorted(repos) @@ -454,7 +457,7 @@ def main() -> int: target_ref = args.branch or detected_branch target_owner = args.owner or detected_owner - branch_name = f"test/starflow-pr-{pr_num}" + branch_name = f"work/bot/starflow-test-pr-{pr_num}" # Determine target repos if args.repos: From 276c935dc24fe183a82df219fce67b9499bf5787 Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 20:58:26 -0400 Subject: [PATCH 3/7] fix: add robust fork handling and error recovery to downstream tester Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/test-downstream.py | 131 +++++++++++++++------- 1 file changed, 91 insertions(+), 40 deletions(-) diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 70f02b2..6bfdee0 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -22,6 +22,12 @@ from pathlib import Path from typing import Any, List, Optional, Tuple +EXCLUDED_REPOS = { + "canonical/starflow", + "canonical/ulwazi", + "canonical/craft-artifacts", +} + DEFAULT_TEAM_REPOS = [ "canonical/snapcraft", "canonical/charmcraft", @@ -32,12 +38,10 @@ "canonical/craft-providers", "canonical/craft-store", "canonical/craft-grammar", - "canonical/craft-artifacts", "canonical/starbase", "canonical/debcraft", "canonical/sdkcraft", "canonical/imagecraft", - "canonical/ulwazi", "canonical/sourcecraft", "canonical/bld", ] @@ -119,8 +123,13 @@ def discover_downstream_repos() -> List[str]: full_name = item.get("full_name") archived = item.get("archived", False) private = item.get("private", False) - # Exclude starflow itself, private repos, or archived repos - if full_name and full_name != "canonical/starflow" and not archived and not private: + # Exclude starflow itself, private repos, archived repos, and excluded repos + if ( + full_name + and full_name not in EXCLUDED_REPOS + and not archived + and not private + ): repos.add(full_name) except Exception: continue @@ -166,6 +175,17 @@ def repl(match: re.Match) -> str: return modified_files +def get_gh_user() -> Optional[str]: + """Get authenticated GitHub username.""" + try: + proc = run_cmd(["gh", "api", "user", "--jq", ".login"], check=False) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip() + except Exception: + pass + return None + + @dataclass class RepoResult: repo: str @@ -199,9 +219,9 @@ def process_repo( with tempfile.TemporaryDirectory(prefix=f"starflow-test-{repo_name}-") as tmpdir: tmp_path = Path(tmpdir) try: - # Shallow clone default branch + # Clone default branch clone_proc = run_cmd( - ["gh", "repo", "clone", repo, str(tmp_path), "--", "--depth=1"], + ["gh", "repo", "clone", repo, str(tmp_path)], check=False, ) if clone_proc.returncode != 0: @@ -249,20 +269,37 @@ def process_repo( ) run_cmd(["git", "commit", "-m", commit_msg], cwd=tmp_path) - # Push branch + # Push branch (try origin first, fall back to personal fork if write access restricted) push_proc = run_cmd(["git", "push", "-u", "origin", branch_name, "--force"], cwd=tmp_path, check=False) + head_spec = branch_name if push_proc.returncode != 0: - return RepoResult( - repo=repo, - status="failed", - modified_files=modified, - error=f"Failed to push branch: {push_proc.stderr.strip()}", - ) + user_login = get_gh_user() + if user_login: + run_cmd(["gh", "repo", "fork", repo, "--default-branch-only", "--clone=false"], check=False) + fork_remote = f"git@github.com:{user_login}/{repo_name}.git" + run_cmd(["git", "remote", "add", "user-fork", fork_remote], cwd=tmp_path, check=False) + push_fork = run_cmd(["git", "push", "-u", "user-fork", branch_name, "--force"], cwd=tmp_path, check=False) + if push_fork.returncode == 0: + head_spec = f"{user_login}:{branch_name}" + else: + return RepoResult( + repo=repo, + status="failed", + modified_files=modified, + error=f"Push failed to origin and fork: {push_fork.stderr.strip()}", + ) + else: + return RepoResult( + repo=repo, + status="failed", + modified_files=modified, + error=f"Failed to push branch: {push_proc.stderr.strip()}", + ) # Check if PR already exists for this branch existing_pr = None pr_view = run_cmd( - ["gh", "pr", "view", branch_name, "--repo", repo, "--json", "url,number"], + ["gh", "pr", "view", head_spec, "--repo", repo, "--json", "url,number"], check=False, ) if pr_view.returncode == 0 and pr_view.stdout.strip(): @@ -272,37 +309,51 @@ def process_repo( except Exception: pass + # Get default branch + default_branch = "main" + def_proc = run_cmd( + ["gh", "repo", "view", repo, "--json", "defaultBranchRef", "--jq", ".defaultBranchRef.name"], + check=False, + ) + if def_proc.returncode == 0 and def_proc.stdout.strip(): + default_branch = def_proc.stdout.strip() + pr_url = None pr_num = None if existing_pr: pr_url, pr_num = existing_pr else: - # Create draft PR - create_pr = run_cmd( - [ - "gh", - "pr", - "create", - "--repo", - repo, - "--draft", - "--title", - title, - "--body", - body, - "--head", - branch_name, - ], - cwd=tmp_path, - check=False, - ) - if create_pr.returncode == 0 and create_pr.stdout.strip(): - pr_url = create_pr.stdout.strip().splitlines()[-1] - # Parse PR number from URL - match = re.search(r"/pull/(\d+)", pr_url) - if match: - pr_num = int(match.group(1)) - else: + # Create draft PR (retry once if ref indexing has a slight delay) + for attempt in range(3): + if attempt > 0: + time.sleep(2.0) + create_pr = run_cmd( + [ + "gh", + "pr", + "create", + "--repo", + repo, + "--draft", + "--title", + title, + "--body", + body, + "--base", + default_branch, + "--head", + head_spec, + ], + cwd=tmp_path, + check=False, + ) + if create_pr.returncode == 0 and create_pr.stdout.strip(): + pr_url = create_pr.stdout.strip().splitlines()[-1] + match = re.search(r"/pull/(\d+)", pr_url) + if match: + pr_num = int(match.group(1)) + break + if not pr_url: return RepoResult( repo=repo, status="failed", From 7df96c949ea3eb2d21bc9d23c145989b4c084b32 Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 21:03:56 -0400 Subject: [PATCH 4/7] feat: disable non-starflow workflows and update existing draft PRs in test-pr skill Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/SKILL.md | 3 +- .github/skills/test-pr/test-downstream.py | 95 ++++++++++++++++++----- 2 files changed, 79 insertions(+), 19 deletions(-) diff --git a/.github/skills/test-pr/SKILL.md b/.github/skills/test-pr/SKILL.md index daf4faf..e547309 100644 --- a/.github/skills/test-pr/SKILL.md +++ b/.github/skills/test-pr/SKILL.md @@ -14,7 +14,8 @@ It automates: 3. For each downstream repository: - Creating a dedicated test branch `work/bot/starflow-test-pr-`. - Updating all `.github/workflows/*.yaml` references to point to the starflow PR branch. - - Pushing the branch and opening a Draft PR titled `[DO NOT MERGE] Test starflow PR `. + - Disabling/removing any workflows that do not consume starflow to prevent unnecessary CI resource and CPU cycle usage. + - Pushing the branch and opening (or updating an existing) Draft PR titled `[DO NOT MERGE] Test starflow PR `. 4. Listing the resulting Draft PRs along with direct links to the triggered GitHub Actions workflow runs. 5. Providing cleanup capabilities once downstream testing is complete. diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 6bfdee0..3008615 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -143,12 +143,16 @@ def discover_downstream_repos() -> List[str]: return sorted(repos) -def update_workflows_in_dir(workflow_dir: Path, target_owner: str, target_ref: str) -> List[str]: +def update_workflows_in_dir( + workflow_dir: Path, target_owner: str, target_ref: str +) -> Tuple[List[str], List[str]]: """ Update workflow files in workflow_dir to point to target_owner/starflow at target_ref. - Returns list of modified file names. + Disables/removes any workflow files that do not use starflow to save CI CPU cycles. + Returns (modified_files, disabled_files). """ modified_files = [] + disabled_files = [] # Pattern to match: uses: /starflow(/path)?@ # Captures: 1=quote if any, 2=subpath if any, 3=old_ref pattern = re.compile( @@ -168,11 +172,16 @@ def repl(match: re.Match) -> str: return f"{prefix}{target_owner}/starflow{subpath}@{target_ref}{suffix}" new_content, count = pattern.subn(repl, content) - if count > 0 and new_content != content: - yaml_path.write_text(new_content, encoding="utf-8") + if count > 0: + if new_content != content: + yaml_path.write_text(new_content, encoding="utf-8") modified_files.append(yaml_path.name) + else: + # File does not use starflow: remove it to prevent triggering unnecessary CI runs + yaml_path.unlink() + disabled_files.append(yaml_path.name) - return modified_files + return modified_files, disabled_files def get_gh_user() -> Optional[str]: @@ -191,6 +200,7 @@ class RepoResult: repo: str status: str modified_files: List[str] + disabled_files: List[str] = None pr_url: Optional[str] = None pr_number: Optional[int] = None branch: Optional[str] = None @@ -209,12 +219,6 @@ def process_repo( """Clone downstream repo, update workflows, push branch and create/update Draft PR.""" repo_name = repo.split("/")[-1] title = f"[DO NOT MERGE] Test starflow PR {pr_number}" - body = ( - f"This is an automated draft PR to test changes from starflow PR #{pr_number}.\n\n" - f"- **Starflow PR**: https://github.com/canonical/starflow/pull/{pr_number}\n" - f"- **Target Ref**: `{target_owner}/starflow@{target_ref}`\n\n" - f"⚠️ **DO NOT MERGE**: This PR is intended solely to verify downstream CI workflows." - ) with tempfile.TemporaryDirectory(prefix=f"starflow-test-{repo_name}-") as tmpdir: tmp_path = Path(tmpdir) @@ -241,31 +245,42 @@ def process_repo( error="No .github/workflows directory found", ) - modified = update_workflows_in_dir(workflow_dir, target_owner, target_ref) + modified, disabled = update_workflows_in_dir(workflow_dir, target_owner, target_ref) if not modified: return RepoResult( repo=repo, status="skipped", modified_files=[], + disabled_files=disabled, error="No starflow workflow references found to update", ) + body = ( + f"This is an automated draft PR to test changes from starflow PR #{pr_number}.\n\n" + f"- **Starflow PR**: https://github.com/canonical/starflow/pull/{pr_number}\n" + f"- **Target Ref**: `{target_owner}/starflow@{target_ref}`\n" + f"- **Starflow Workflows Tested**: {', '.join(modified)}\n" + f"- **Disabled Non-Starflow Workflows**: {', '.join(disabled) if disabled else 'none'}\n\n" + f"⚠️ **DO NOT MERGE**: This PR is intended solely to verify downstream CI workflows." + ) + if dry_run: return RepoResult( repo=repo, status="dry-run", modified_files=modified, + disabled_files=disabled, branch=branch_name, ) # Create branch, commit, push run_cmd(["git", "checkout", "-b", branch_name], cwd=tmp_path) - for m in modified: - run_cmd(["git", "add", f".github/workflows/{m}"], cwd=tmp_path) + run_cmd(["git", "add", "-A", ".github/workflows"], cwd=tmp_path) commit_msg = ( f"ci: test against starflow PR #{pr_number} ({target_ref})\n\n" - f"[DO NOT MERGE] Point starflow workflows to {target_owner}/starflow@{target_ref}" + f"[DO NOT MERGE] Point starflow workflows to {target_owner}/starflow@{target_ref}\n" + f"Disabled non-starflow workflows: {', '.join(disabled) if disabled else 'none'}" ) run_cmd(["git", "commit", "-m", commit_msg], cwd=tmp_path) @@ -296,19 +311,45 @@ def process_repo( error=f"Failed to push branch: {push_proc.stderr.strip()}", ) - # Check if PR already exists for this branch + # Check if an open PR already exists for testing this starflow PR existing_pr = None pr_view = run_cmd( - ["gh", "pr", "view", head_spec, "--repo", repo, "--json", "url,number"], + ["gh", "pr", "view", head_spec, "--repo", repo, "--json", "url,number,state"], check=False, ) if pr_view.returncode == 0 and pr_view.stdout.strip(): try: pr_data = json.loads(pr_view.stdout) - existing_pr = (pr_data.get("url"), pr_data.get("number")) + if pr_data.get("state") == "OPEN": + existing_pr = (pr_data.get("url"), pr_data.get("number")) except Exception: pass + if not existing_pr: + pr_list_proc = run_cmd( + [ + "gh", + "pr", + "list", + "--repo", + repo, + "--state", + "open", + "--head", + branch_name, + "--json", + "url,number", + ], + check=False, + ) + if pr_list_proc.returncode == 0 and pr_list_proc.stdout.strip(): + try: + pr_list_data = json.loads(pr_list_proc.stdout) + if pr_list_data: + existing_pr = (pr_list_data[0].get("url"), pr_list_data[0].get("number")) + except Exception: + pass + # Get default branch default_branch = "main" def_proc = run_cmd( @@ -322,6 +363,23 @@ def process_repo( pr_num = None if existing_pr: pr_url, pr_num = existing_pr + # Update title and body of existing PR + run_cmd( + [ + "gh", + "pr", + "edit", + str(pr_num), + "--repo", + repo, + "--title", + title, + "--body", + body, + ], + cwd=tmp_path, + check=False, + ) else: # Create draft PR (retry once if ref indexing has a slight delay) for attempt in range(3): @@ -358,6 +416,7 @@ def process_repo( repo=repo, status="failed", modified_files=modified, + disabled_files=disabled, error=f"Failed to create draft PR: {create_pr.stderr.strip()}", ) From cac9b22e30b2816a5b2e2e9556b1ba857e56dc88 Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 21:07:51 -0400 Subject: [PATCH 5/7] feat: parallelize downstream PR creation in test-pr skill Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/test-downstream.py | 86 ++++++++++++++--------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 3008615..2f4222b 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -10,6 +10,7 @@ from __future__ import annotations import argparse +import concurrent.futures import json import os import re @@ -475,11 +476,11 @@ def fetch_workflow_runs(repo: str, branch: str, retries: int = 3, delay: float = return runs -def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str) -> None: - """Close test draft PRs and delete branches across repos.""" - print(f"Cleaning up test PRs and branches for starflow PR #{pr_number}...") - for repo in repos: - print(f"\n[{repo}]") +def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str, workers: int = 8) -> None: + """Close test draft PRs and delete branches across repos in parallel.""" + print(f"Cleaning up test PRs and branches for starflow PR #{pr_number} ({len(repos)} repositories, workers={workers})...\n") + + def _cleanup_one(repo: str) -> None: # Check for open PR on branch proc = run_cmd( ["gh", "pr", "view", branch_name, "--repo", repo, "--json", "number,url,state"], @@ -491,10 +492,10 @@ def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str) -> None pr_num = data.get("number") state = data.get("state") if pr_num and state == "OPEN": - print(f" Closing draft PR #{pr_num}...") + print(f"[{repo}] Closing draft PR #{pr_num}...") run_cmd(["gh", "pr", "close", str(pr_num), "--repo", repo, "--delete-branch"], check=False) - print(f" Closed #{pr_num} and requested branch deletion.") - continue + print(f"[{repo}] Closed #{pr_num} and deleted branch.") + return except Exception: pass @@ -504,14 +505,17 @@ def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str) -> None check=False, ) if del_proc.returncode == 0: - print(f" Deleted branch {branch_name}.") + print(f"[{repo}] Deleted branch {branch_name}.") else: - print(" No branch or PR found to clean up.") + print(f"[{repo}] No branch or PR found to clean up.") + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + list(executor.map(_cleanup_one, repos)) def main() -> int: parser = argparse.ArgumentParser( - description="Test starflow PRs across downstream repositories by creating draft PRs." + description="Test starflow PRs across downstream repositories by creating draft PRs in parallel." ) parser.add_argument( "--pr", @@ -535,6 +539,13 @@ def main() -> int: nargs="+", help="Specific downstream repos to test (e.g. canonical/snapcraft). Defaults to all team repos using starflow.", ) + parser.add_argument( + "--workers", + "-w", + type=int, + default=8, + help="Number of concurrent workers for parallel cloning and PR creation (default: 8).", + ) parser.add_argument( "--dry-run", action="store_true", @@ -584,7 +595,7 @@ def main() -> int: repos = discover_downstream_repos() if args.cleanup: - cleanup_test_prs(repos, pr_num, branch_name) + cleanup_test_prs(repos, pr_num, branch_name, workers=args.workers) return 0 print(f"==================================================") @@ -594,32 +605,41 @@ def main() -> int: print(f" Test Branch : {branch_name}") print(f" PR Title : [DO NOT MERGE] Test starflow PR {pr_num}") print(f" Repositories: {len(repos)} candidate repo(s)") + print(f" Concurrency : {args.workers} workers") if args.dry_run: print(f" Mode : DRY RUN (no git push or PR creation)") print(f"==================================================\n") results: List[RepoResult] = [] - - for idx, repo in enumerate(repos, 1): - print(f"[{idx}/{len(repos)}] Processing {repo}...") - res = process_repo( - repo=repo, - pr_number=pr_num, - target_owner=target_owner, - target_ref=target_ref, - branch_name=branch_name, - dry_run=args.dry_run, - ) - results.append(res) - if res.status == "success": - print(f" ✓ Draft PR created: {res.pr_url}") - print(f" Modified workflows: {', '.join(res.modified_files)}") - elif res.status == "dry-run": - print(f" ✓ [Dry Run] Would modify: {', '.join(res.modified_files)}") - elif res.status == "skipped": - print(f" - Skipped: {res.error}") - else: - print(f" ✗ Failed: {res.error}") + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: + future_to_repo = { + executor.submit( + process_repo, + repo=repo, + pr_number=pr_num, + target_owner=target_owner, + target_ref=target_ref, + branch_name=branch_name, + dry_run=args.dry_run, + ): repo + for repo in repos + } + for future in concurrent.futures.as_completed(future_to_repo): + repo = future_to_repo[future] + try: + res = future.result() + results.append(res) + if res.status == "success": + print(f" ✓ [{res.repo}] Draft PR ready: {res.pr_url} (tested: {', '.join(res.modified_files)})") + elif res.status == "dry-run": + print(f" ✓ [{res.repo}] [Dry Run] Would modify: {', '.join(res.modified_files)}") + elif res.status == "skipped": + print(f" - [{res.repo}] Skipped: {res.error}") + else: + print(f" ✗ [{res.repo}] Failed: {res.error}") + except Exception as e: + print(f" ✗ [{repo}] Unexpected error: {e}") + results.append(RepoResult(repo=repo, status="failed", modified_files=[], error=str(e))) if args.json: out_data = [ From 6e5f372c1b6214c95664bc5d236f715b3570858f Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 21:13:22 -0400 Subject: [PATCH 6/7] feat: disable non-starflow jobs using if: false instead of deleting files Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/SKILL.md | 30 ++-- .github/skills/test-pr/test-downstream.py | 208 +++++++++++++++++----- 2 files changed, 179 insertions(+), 59 deletions(-) diff --git a/.github/skills/test-pr/SKILL.md b/.github/skills/test-pr/SKILL.md index e547309..f86a7f2 100644 --- a/.github/skills/test-pr/SKILL.md +++ b/.github/skills/test-pr/SKILL.md @@ -14,7 +14,7 @@ It automates: 3. For each downstream repository: - Creating a dedicated test branch `work/bot/starflow-test-pr-`. - Updating all `.github/workflows/*.yaml` references to point to the starflow PR branch. - - Disabling/removing any workflows that do not consume starflow to prevent unnecessary CI resource and CPU cycle usage. + - Disabling non-starflow jobs using `if: false` to prevent unnecessary CI resource and CPU cycle usage without deleting files. - Pushing the branch and opening (or updating an existing) Draft PR titled `[DO NOT MERGE] Test starflow PR `. 4. Listing the resulting Draft PRs along with direct links to the triggered GitHub Actions workflow runs. 5. Providing cleanup capabilities once downstream testing is complete. @@ -49,8 +49,11 @@ Identify the starflow PR number and branch to test: Run the helper script from the repository root: ```bash -# Test across all downstream repositories -python3 .github/skills/test-pr/test-downstream.py --pr +# Test across all downstream repositories in parallel +python3 .github/skills/test-pr/test-downstream.py --pr --workers 16 + +# Test and automatically update the starflow PR description with the results table +python3 .github/skills/test-pr/test-downstream.py --pr --update-pr-body # Test with an explicit branch/ref or fork owner python3 .github/skills/test-pr/test-downstream.py --pr --branch --owner @@ -62,25 +65,24 @@ python3 .github/skills/test-pr/test-downstream.py --pr --repos canon python3 .github/skills/test-pr/test-downstream.py --pr --dry-run ``` -### 3. Present Results to the User +### 3. Present Results & Offer PR Body Update -Present the results clearly in a table and list, including: +Present the results to the user: -- Repository name -- Draft PR link (titled `[DO NOT MERGE] Test starflow PR ###`) -- Direct links to triggered GitHub Actions workflow runs -- Actions overview URL for each repository +1. Display the rendered summary table with direct links to draft PRs and test workflow runs. +2. Provide a copy-pasteable raw markdown block containing the table. +3. Offer the user the choice to either copy the raw markdown or have the agent update the Starflow PR description automatically (or pass `--update-pr-body`). Example output format: ```markdown ### Downstream Test PRs for Starflow PR #42 -| Repository | Draft PR | Test Workflow Runs | -| :----------------------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **canonical/snapcraft** | [#123](https://github.com/canonical/snapcraft/pull/123) | [QA](https://github.com/canonical/snapcraft/actions/runs/1001) (in_progress)
[TICS](https://github.com/canonical/snapcraft/actions/runs/1002) (in_progress) | -| **canonical/charmcraft** | [#456](https://github.com/canonical/charmcraft/pull/456) | [QA](https://github.com/canonical/charmcraft/actions/runs/1003) (in_progress) | -| **canonical/rockcraft** | [#789](https://github.com/canonical/rockcraft/pull/789) | [QA](https://github.com/canonical/rockcraft/actions/runs/1004) (in_progress) | +| Repository | Draft PR | Test Workflow Runs | +| :----------------------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------- | +| **canonical/snapcraft** | [#123](https://github.com/canonical/snapcraft/pull/123) | [QA](https://github.com/canonical/snapcraft/actions/runs/1001), [Check policy](https://github.com/canonical/snapcraft/actions/runs/1002) | +| **canonical/charmcraft** | [#456](https://github.com/canonical/charmcraft/pull/456) | [QA](https://github.com/canonical/charmcraft/actions/runs/1003), [Check policy](https://github.com/canonical/charmcraft/actions/runs/1004) | +| **canonical/rockcraft** | [#789](https://github.com/canonical/rockcraft/pull/789) | [QA](https://github.com/canonical/rockcraft/actions/runs/1005), [Check policy](https://github.com/canonical/rockcraft/actions/runs/1006) | ``` ### 4. Cleanup diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 2f4222b..3851505 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -23,6 +23,8 @@ from pathlib import Path from typing import Any, List, Optional, Tuple +import yaml + EXCLUDED_REPOS = { "canonical/starflow", "canonical/ulwazi", @@ -144,45 +146,107 @@ def discover_downstream_repos() -> List[str]: return sorted(repos) +def update_single_workflow_text( + content: str, target_owner: str, target_ref: str +) -> Tuple[str, bool, List[str]]: + """ + Update a single workflow YAML content. + - Updates starflow references to target_owner/starflow@target_ref. + - Adds `if: false` to non-starflow jobs to prevent unnecessary CI execution. + Returns (new_content, uses_starflow, disabled_job_ids). + """ + try: + data = yaml.safe_load(content) + except Exception: + data = None + + if not isinstance(data, dict) or "jobs" not in data or not isinstance(data.get("jobs"), dict): + return content, False, [] + + jobs = data.get("jobs", {}) + starflow_jobs = set() + non_starflow_jobs = set() + + for job_id, job_info in jobs.items(): + if not isinstance(job_info, dict): + continue + uses_sf = False + if "uses" in job_info and "starflow" in str(job_info["uses"]): + uses_sf = True + for step in job_info.get("steps") or []: + if isinstance(step, dict) and "uses" in step and "starflow" in str(step.get("uses")): + uses_sf = True + break + if uses_sf: + starflow_jobs.add(str(job_id)) + else: + non_starflow_jobs.add(str(job_id)) + + # 1. Update starflow references in text + pattern = re.compile( + r'(uses:\s*[\'"]?)[a-zA-Z0-9_.-]+/starflow(/[^@\s\r\n\'"]*)?@([^\s\r\n\'"]+)([\'"]?)' + ) + new_content = pattern.sub(rf"\g<1>{target_owner}/starflow\g<2>@{target_ref}\g<4>", content) + + # 2. Add `if: false` to non-starflow jobs + lines = new_content.splitlines() + in_jobs = False + current_job = None + output_lines = [] + + for line in lines: + if re.match(r"^jobs:\s*(?:#.*)?$", line): + in_jobs = True + output_lines.append(line) + continue + if in_jobs: + job_match = re.match(r"^ ([a-zA-Z0-9_.-]+):\s*(?:#.*)?$", line) + if job_match: + current_job = job_match.group(1) + output_lines.append(line) + if current_job in non_starflow_jobs: + output_lines.append(" if: false") + continue + elif re.match(r"^\S", line): + in_jobs = False + current_job = None + elif current_job in non_starflow_jobs and re.match(r"^\s+if:\s*", line): + continue + output_lines.append(line) + + final_content = "\n".join(output_lines) + ("\n" if content.endswith("\n") else "") + return final_content, len(starflow_jobs) > 0, sorted(non_starflow_jobs) + + def update_workflows_in_dir( workflow_dir: Path, target_owner: str, target_ref: str ) -> Tuple[List[str], List[str]]: """ Update workflow files in workflow_dir to point to target_owner/starflow at target_ref. - Disables/removes any workflow files that do not use starflow to save CI CPU cycles. - Returns (modified_files, disabled_files). + Disables non-starflow jobs using `if: false` to save CI CPU cycles. + Returns (starflow_workflow_files, disabled_jobs_summary). """ - modified_files = [] - disabled_files = [] - # Pattern to match: uses: /starflow(/path)?@ - # Captures: 1=quote if any, 2=subpath if any, 3=old_ref - pattern = re.compile( - r'(uses:\s*[\'"]?)[a-zA-Z0-9_.-]+/starflow(/[^@\s\r\n\'"]*)?@([^\s\r\n\'"]+)([\'"]?)' - ) + starflow_files = [] + disabled_jobs_summary = [] - for yaml_path in list(workflow_dir.glob("*.yaml")) + list(workflow_dir.glob("*.yml")): + for yaml_path in sorted(list(workflow_dir.glob("*.yaml")) + list(workflow_dir.glob("*.yml"))): try: content = yaml_path.read_text(encoding="utf-8") except Exception: continue - def repl(match: re.Match) -> str: - prefix = match.group(1) - subpath = match.group(2) or "" - suffix = match.group(4) or "" - return f"{prefix}{target_owner}/starflow{subpath}@{target_ref}{suffix}" - - new_content, count = pattern.subn(repl, content) - if count > 0: - if new_content != content: - yaml_path.write_text(new_content, encoding="utf-8") - modified_files.append(yaml_path.name) - else: - # File does not use starflow: remove it to prevent triggering unnecessary CI runs - yaml_path.unlink() - disabled_files.append(yaml_path.name) + new_content, uses_sf, disabled_jobs = update_single_workflow_text( + content, target_owner, target_ref + ) + if uses_sf: + starflow_files.append(yaml_path.name) + if disabled_jobs: + disabled_jobs_summary.append(f"{yaml_path.name} ({', '.join(disabled_jobs)})") + + if new_content != content: + yaml_path.write_text(new_content, encoding="utf-8") - return modified_files, disabled_files + return starflow_files, disabled_jobs_summary def get_gh_user() -> Optional[str]: @@ -261,7 +325,7 @@ def process_repo( f"- **Starflow PR**: https://github.com/canonical/starflow/pull/{pr_number}\n" f"- **Target Ref**: `{target_owner}/starflow@{target_ref}`\n" f"- **Starflow Workflows Tested**: {', '.join(modified)}\n" - f"- **Disabled Non-Starflow Workflows**: {', '.join(disabled) if disabled else 'none'}\n\n" + f"- **Disabled Non-Starflow Jobs (`if: false`)**: {', '.join(disabled) if disabled else 'none'}\n\n" f"⚠️ **DO NOT MERGE**: This PR is intended solely to verify downstream CI workflows." ) @@ -281,7 +345,7 @@ def process_repo( commit_msg = ( f"ci: test against starflow PR #{pr_number} ({target_ref})\n\n" f"[DO NOT MERGE] Point starflow workflows to {target_owner}/starflow@{target_ref}\n" - f"Disabled non-starflow workflows: {', '.join(disabled) if disabled else 'none'}" + f"Disabled non-starflow jobs (if: false): {', '.join(disabled) if disabled else 'none'}" ) run_cmd(["git", "commit", "-m", commit_msg], cwd=tmp_path) @@ -476,6 +540,55 @@ def fetch_workflow_runs(repo: str, branch: str, retries: int = 3, delay: float = return runs +def format_markdown_table(successful: List[RepoResult], branch_name: str) -> str: + """Format successful downstream test PR results into a markdown table.""" + lines = [ + "| Repository | Draft PR | Test Workflow Runs |", + "| :--- | :--- | :--- |", + ] + for r in sorted(successful, key=lambda x: x.repo): + runs_links = [] + if r.workflow_runs: + for run in r.workflow_runs: + name = run.get("name", "Workflow") + url = run.get("url", "") + runs_links.append(f"[{name}]({url})") + if not runs_links: + actions_url = f"https://github.com/{r.repo}/actions?query=branch%3A{branch_name}" + runs_links.append(f"[All Action Runs]({actions_url})") + + runs_str = ", ".join(runs_links) + lines.append(f"| **{r.repo}** | [#{r.pr_number}]({r.pr_url}) | {runs_str} |") + return "\n".join(lines) + + +def update_starflow_pr_description(pr_number: str, table_md: str, owner: str = "canonical") -> bool: + """Update or append the downstream test results table in the starflow PR description.""" + try: + proc = run_cmd( + ["gh", "pr", "view", str(pr_number), "--repo", f"{owner}/starflow", "--json", "body", "--jq", ".body"], + check=False, + ) + if proc.returncode != 0: + return False + current_body = proc.stdout.strip() + + section_header = "### Downstream Test PRs" + if section_header in current_body: + pattern = re.compile(r"### Downstream Test PRs.*?(?=\n### |\Z)", re.DOTALL) + new_body = pattern.sub(f"{section_header}\n\n{table_md}", current_body).strip() + else: + new_body = f"{current_body}\n\n{section_header}\n\n{table_md}".strip() + + edit_proc = run_cmd( + ["gh", "pr", "edit", str(pr_number), "--repo", f"{owner}/starflow", "--body", new_body], + check=False, + ) + return edit_proc.returncode == 0 + except Exception: + return False + + def cleanup_test_prs(repos: List[str], pr_number: str, branch_name: str, workers: int = 8) -> None: """Close test draft PRs and delete branches across repos in parallel.""" print(f"Cleaning up test PRs and branches for starflow PR #{pr_number} ({len(repos)} repositories, workers={workers})...\n") @@ -551,6 +664,11 @@ def main() -> int: action="store_true", help="Simulate workflow changes without pushing branches or creating PRs.", ) + parser.add_argument( + "--update-pr-body", + action="store_true", + help="Automatically update/append the downstream test table into the Starflow PR description.", + ) parser.add_argument( "--cleanup", action="store_true", @@ -668,25 +786,25 @@ def main() -> int: print("=" * 50 + "\n") if successful: - print("| Repository | Draft PR | Workflow Runs / Actions |") - print("| :--- | :--- | :--- |") - for r in successful: - runs_links = [] - if r.workflow_runs: - for run in r.workflow_runs: - name = run.get("name", "Workflow") - url = run.get("url", "") - status = run.get("status", "") - runs_links.append(f"[{name}]({url}) ({status})") - if not runs_links: - actions_url = f"https://github.com/{r.repo}/actions?query=branch%3A{branch_name}" - runs_links.append(f"[All Action Runs]({actions_url})") - - runs_str = "
".join(runs_links) - print(f"| **{r.repo}** | [{r.repo}#{r.pr_number}]({r.pr_url}) | {runs_str} |") + table_md = format_markdown_table(successful, branch_name) + print("### Rendered Table:\n") + print(table_md) + + print("\n### Raw Markdown (Copy & Paste):\n") + print("```markdown") + print("### Downstream Test PRs\n") + print(table_md) + print("```") + + if args.update_pr_body: + print("\nUpdating Starflow PR description...") + if update_starflow_pr_description(pr_num, table_md, target_owner): + print(f"✓ Updated Starflow PR #{pr_num} description with downstream test table.") + else: + print(f"✗ Failed to update Starflow PR #{pr_num} description.") print("\n### Direct Links:") - for r in successful: + for r in sorted(successful, key=lambda x: x.repo): print(f"\n#### {r.repo}") print(f"- **Draft PR**: {r.pr_url}") if r.workflow_runs: From c442f3ca78e5cbb58a8d84e0c0edd8307539a1ba Mon Sep 17 00:00:00 2001 From: "Alex M. Lowe" Date: Wed, 14 Oct 2026 21:19:45 -0400 Subject: [PATCH 7/7] fix(skills): address review feedback on downstream test helper - Use jq line emission for paginated repository discovery - Default --owner to None to allow detected fork owner fallback - Resolve headRefName and owner when --pr is specified from another branch - Route progress and log messages to stderr when --json is enabled Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/test-pr/test-downstream.py | 116 ++++++++++++++-------- 1 file changed, 74 insertions(+), 42 deletions(-) diff --git a/.github/skills/test-pr/test-downstream.py b/.github/skills/test-pr/test-downstream.py index 3851505..42cb580 100755 --- a/.github/skills/test-pr/test-downstream.py +++ b/.github/skills/test-pr/test-downstream.py @@ -105,37 +105,55 @@ def get_current_starflow_pr_or_branch() -> Tuple[Optional[str], str, str]: return None, "main", "canonical" +def get_starflow_pr_info( + pr_number: str, owner: str = "canonical" +) -> Tuple[Optional[str], Optional[str]]: + """Fetch headRefName and headRepositoryOwner login for a given Starflow PR number.""" + try: + proc = run_cmd( + [ + "gh", + "pr", + "view", + str(pr_number), + "--repo", + f"{owner}/starflow", + "--json", + "headRefName,headRepositoryOwner", + ], + check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + data = json.loads(proc.stdout) + branch = data.get("headRefName") + repo_owner = data.get("headRepositoryOwner", {}).get("login") + return branch, repo_owner + except Exception: + pass + return None, None + + def discover_downstream_repos() -> List[str]: """Find team repositories in canonical org that use starflow in workflows.""" repos = set() - # 1. Query starcraft team repos + # 1. Query starcraft team repos via jq for robust line-by-line parsing try: proc = run_cmd( - ["gh", "api", "orgs/canonical/teams/starcraft/repos", "--paginate"], + [ + "gh", + "api", + "orgs/canonical/teams/starcraft/repos", + "--paginate", + "--jq", + ".[] | select(.archived == false and .private == false) | .full_name", + ], check=False, ) if proc.returncode == 0 and proc.stdout.strip(): for line in proc.stdout.splitlines(): - if not line.strip(): - continue - try: - data = json.loads(line) - items = data if isinstance(data, list) else [data] - for item in items: - if isinstance(item, dict): - full_name = item.get("full_name") - archived = item.get("archived", False) - private = item.get("private", False) - # Exclude starflow itself, private repos, archived repos, and excluded repos - if ( - full_name - and full_name not in EXCLUDED_REPOS - and not archived - and not private - ): - repos.add(full_name) - except Exception: - continue + full_name = line.strip() + if full_name and full_name not in EXCLUDED_REPOS: + repos.add(full_name) except Exception: pass @@ -643,8 +661,8 @@ def main() -> int: parser.add_argument( "--owner", "-o", - default="canonical", - help="Repository owner/fork hosting the starflow branch (default: canonical).", + default=None, + help="Repository owner/fork hosting the starflow branch (default: auto-detected or canonical).", ) parser.add_argument( "--repos", @@ -682,6 +700,12 @@ def main() -> int: args = parser.parse_args() + def log(msg: str = "") -> None: + if args.json: + print(msg, file=sys.stderr) + else: + print(msg) + # Determine PR number, branch, owner detected_pr, detected_branch, detected_owner = get_current_starflow_pr_or_branch() @@ -694,8 +718,15 @@ def main() -> int: print("Error: Could not determine starflow PR number. Please pass --pr .", file=sys.stderr) return 1 - target_ref = args.branch or detected_branch - target_owner = args.owner or detected_owner + pr_head_branch = None + pr_head_owner = None + if pr_num: + pr_head_branch, pr_head_owner = get_starflow_pr_info( + pr_num, owner=args.owner or detected_owner or "canonical" + ) + + target_ref = args.branch or pr_head_branch or detected_branch or "main" + target_owner = args.owner or pr_head_owner or detected_owner or "canonical" branch_name = f"work/bot/starflow-test-pr-{pr_num}" # Determine target repos @@ -709,24 +740,24 @@ def main() -> int: item = f"canonical/{item}" repos.append(item) else: - print("Discovering downstream repositories using starflow...") + log("Discovering downstream repositories using starflow...") repos = discover_downstream_repos() if args.cleanup: cleanup_test_prs(repos, pr_num, branch_name, workers=args.workers) return 0 - print(f"==================================================") - print(f"Starflow PR Downstream Testing") - print(f" PR Number : #{pr_num}") - print(f" Target Ref : {target_owner}/starflow@{target_ref}") - print(f" Test Branch : {branch_name}") - print(f" PR Title : [DO NOT MERGE] Test starflow PR {pr_num}") - print(f" Repositories: {len(repos)} candidate repo(s)") - print(f" Concurrency : {args.workers} workers") + log("==================================================") + log("Starflow PR Downstream Testing") + log(f" PR Number : #{pr_num}") + log(f" Target Ref : {target_owner}/starflow@{target_ref}") + log(f" Test Branch : {branch_name}") + log(f" PR Title : [DO NOT MERGE] Test starflow PR {pr_num}") + log(f" Repositories: {len(repos)} candidate repo(s)") + log(f" Concurrency : {args.workers} workers") if args.dry_run: - print(f" Mode : DRY RUN (no git push or PR creation)") - print(f"==================================================\n") + log(" Mode : DRY RUN (no git push or PR creation)") + log("==================================================\n") results: List[RepoResult] = [] with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: @@ -748,15 +779,15 @@ def main() -> int: res = future.result() results.append(res) if res.status == "success": - print(f" ✓ [{res.repo}] Draft PR ready: {res.pr_url} (tested: {', '.join(res.modified_files)})") + log(f" ✓ [{res.repo}] Draft PR ready: {res.pr_url} (tested: {', '.join(res.modified_files)})") elif res.status == "dry-run": - print(f" ✓ [{res.repo}] [Dry Run] Would modify: {', '.join(res.modified_files)}") + log(f" ✓ [{res.repo}] [Dry Run] Would modify: {', '.join(res.modified_files)}") elif res.status == "skipped": - print(f" - [{res.repo}] Skipped: {res.error}") + log(f" - [{res.repo}] Skipped: {res.error}") else: - print(f" ✗ [{res.repo}] Failed: {res.error}") + log(f" ✗ [{res.repo}] Failed: {res.error}") except Exception as e: - print(f" ✗ [{repo}] Unexpected error: {e}") + log(f" ✗ [{repo}] Unexpected error: {e}") results.append(RepoResult(repo=repo, status="failed", modified_files=[], error=str(e))) if args.json: @@ -765,6 +796,7 @@ def main() -> int: "repo": r.repo, "status": r.status, "modified_files": r.modified_files, + "disabled_files": r.disabled_files or [], "pr_url": r.pr_url, "pr_number": r.pr_number, "branch": r.branch,