diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6210deb..87c842b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,7 +6,7 @@ }, "metadata": { "description": "Automation skills for GitHub org maintenance", - "version": "1.0.0" + "version": "0.1.0" }, "plugins": [ { @@ -95,7 +95,15 @@ "source": "./", "strict": false, "skills": [ - "./skills/repoman" + "./skills/repoman", + "./skills/link-health-scanner", + "./skills/link-health-fixer", + "./skills/dep-bump-scanner", + "./skills/dep-bump-fixer", + "./skills/github-pr-review", + "./skills/repo-sync", + "./skills/automation-health-dashboard", + "./skills/github-weekly-report" ] } ] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..604f331 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + +All notable changes to this project are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres +to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-10-23 + +First tagged release of RepoMan, distributed through the `rossoctl-agent-skills` +marketplace. + +RepoMan runs automated maintenance **programs** across a set of GitHub repos you +enroll once, handling the capability checks and output wiring for you. `/repoman` is +a single intent-dispatching entrypoint that infers whether you are setting up, +enabling a program, or running one. + +### Added + +- **Self-contained `repoman` plugin.** One install bundles the `/repoman` entrypoint, + every program skill, and the engine scripts, so you install once and enable more + programs on the fly. The engine is vendored into the plugin and resolved at runtime + via `${CLAUDE_PLUGIN_ROOT}` — no separate automation checkout is required. +- **Six programs** ship ready to enable: `link-health` (scanner + fixer), + `dep-bump` (scanner + fixer), `pr-review`, `repo-sync`, `automation-health`, and + `weekly-report`. +- **Standalone program plugins** remain installable on their own, for using a single + program (for example a one-off PR review) without the full RepoMan bundle. +- **Provenance stamp** (`_vendor.json`) records the exact `rossoctl/automation` commit + the bundled engine was vendored from, so a shipped bundle is traceable to its source. + +### Changed + +- `marketplace.json` version reconciled to `0.1.0`. +- All skills now declare `bash` 3.2+ and run on the macOS system default. + +### Notes + +- Engine development lives in `rossoctl/automation`; engine fixes reach every RepoMan + user through a single `repoman` plugin update. + +### Contributors + +Gloire Rubambiza, Aleksander Slominski, Mariusz Sabath, Jim Cadden. + +[0.1.0]: https://github.com/rossoctl/agent-skills/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 6e915d1..2c80e06 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Built to the [Agent Skills specification](https://agentskills.io/specification). | Skill | Description | |-------|-------------| -| [repoman](skills/repoman/) | The RepoMan entrypoint — first-time setup, enabling a program, enrolling repos or an org, or running a program across enrolled repos; a single intent-dispatching skill | +| [repoman](skills/repoman/) | The RepoMan entrypoint and self-contained bundle — first-time setup, enabling a program, enrolling repos or an org, running a program; ships every program skill + the engine so you install once and enable programs on the fly | | [link-health-scanner](skills/link-health-scanner/) | Scan repos for broken links, create GitHub issues for findings | | [link-health-fixer](skills/link-health-fixer/) | Re-verify and fix broken links, open PRs for fixes | | [dep-bump-scanner](skills/dep-bump-scanner/) | Monitor Dependabot PRs, classify by severity, flag SLA breaches | @@ -29,14 +29,24 @@ Built to the [Agent Skills specification](https://agentskills.io/specification). /plugin install @rossoctl-agent-skills ``` -Install one plugin at a time. Replace `` with a plugin name such as -`repoman`, `link-health`, `dep-bump`, `pr-review`, `automation-health`, or -`repo-sync`. Plugin names are the ones declared in the marketplace, which can -differ from the per-skill directory names in the [Skills](#skills) table. For -example, the `link-health` plugin bundles the `link-health-scanner` and -`link-health-fixer` skills. Run `/plugin` after adding the marketplace to browse -the authoritative list. Installing one plugin does **not** pull in the others, -so repeat the `install` line per plugin you want. +**Recommended: install `repoman`.** The `repoman` plugin is self-contained — it +bundles the RepoMan entrypoint, every program skill (link-health, dep-bump, +pr-review, repo-sync, automation-health, weekly-report) with their scanner and +fixer skills, and the engine scripts. One install gives you the whole system and +lets you enable more programs on the fly: + +``` +/plugin install repoman@rossoctl-agent-skills +``` + +**Or install a single program standalone.** Each program is also published as its +own plugin, for using it on its own (e.g. a one-off PR review) without the full +RepoMan bundle. Replace `` with a plugin name such as `link-health`, +`dep-bump`, `pr-review`, `automation-health`, or `repo-sync`. Plugin names are the +ones declared in the marketplace, which can differ from the per-skill directory +names in the [Skills](#skills) table. Run `/plugin` after adding the marketplace to +browse the authoritative list. Installing one standalone plugin does **not** pull +in the others, so repeat the `install` line per plugin you want. ### Updating @@ -48,6 +58,9 @@ update any installed plugins (a Claude Code restart applies plugin updates): /plugin update @rossoctl-agent-skills ``` +Updating `repoman` refreshes the bundled engine scripts too, so engine fixes reach +every RepoMan user through a single plugin update. + ### Manual Copy the desired skill directory into your project's `.claude/skills/` directory: @@ -60,7 +73,8 @@ cp -r skills/link-health-scanner /path/to/project/.claude/skills/ All skills in this repo require: -- `bash` 4+ +- `bash` 3.2+ — every skill runs on the macOS system default; no bash-4 features + are used. - `gh` (GitHub CLI, authenticated) - `jq` diff --git a/scripts/vendor-engine.sh b/scripts/vendor-engine.sh new file mode 100644 index 0000000..2a0b922 --- /dev/null +++ b/scripts/vendor-engine.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -uo pipefail +# Vendor the RepoMan engine from an automation checkout into the repoman plugin's +# flat scripts dir, then stamp the source commit SHA for provenance. +# +# Explicit manifest (never a glob) so this can only ever populate the repoman +# plugin — it must never touch a standalone skill's scripts/ (see agent-skills#51). +# Part of the RepoMan v0.1.0 packaged release (epic rossoctl/automation#72). +# +# vendor-engine.sh --source --dest + +SRC="" DEST="" +while [ $# -gt 0 ]; do + case "$1" in + --source) SRC="${2:-}"; shift 2 ;; + --dest) DEST="${2:-}"; shift 2 ;; + *) echo "ERROR: unknown arg: $1" >&2; exit 2 ;; + esac +done +[ -n "$SRC" ] && [ -n "$DEST" ] || { echo "ERROR: --source and --dest are required" >&2; exit 2; } + +# The 23 engine scripts. Flat set: nearly all source program-lib.sh, +# which sources core.sh/github-api.sh/fork.sh/org.sh/repoman-requirements.sh. +MANIFEST="automation-health-dashboard core dep-bump-fixer dep-bump-scanner \ +extract-broken-links fork github-api link-health-fixer link-health-scanner \ +org parse-diff-map pr-review-fixer pr-review-impact pr-review-lib \ +pr-review-scanner program-lib repo-sync repoman-check repoman-index \ +repoman-requirements repoman-run repoman-setup weekly-report" + +# Pre-validate the WHOLE manifest before copying anything, so a source missing +# late in the list cannot leave a half-vendored dest with a stale _vendor.json. +# shellcheck disable=SC2086 +for f in $MANIFEST; do + if [ ! -f "$SRC/scripts/$f.sh" ]; then + echo "ERROR: missing engine source: $SRC/scripts/$f.sh (nothing copied)" >&2; exit 1 + fi +done + +mkdir -p "$DEST" +# shellcheck disable=SC2086 +for f in $MANIFEST; do + if ! cp "$SRC/scripts/$f.sh" "$DEST/$f.sh"; then + echo "ERROR: copy failed for $f.sh" >&2; exit 1 + fi +done + +# Provenance stamp: record the SOURCE repo's HEAD sha (not the dest repo's). +if ! SHA=$(git -C "$SRC" rev-parse HEAD 2>/dev/null); then + echo "ERROR: cannot read source HEAD sha from $SRC" >&2; exit 1 +fi +NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# word-splitting $MANIFEST into one line per script is intentional +# shellcheck disable=SC2086 +if ! FILES_JSON=$(printf '%s\n' $MANIFEST | sed 's/$/.sh/' | jq -R . | jq -s .); then + echo "ERROR: could not build file list" >&2; exit 1 +fi +if ! jq -n --arg repo "rossoctl/automation" --arg sha "$SHA" --arg at "$NOW" \ + --argjson files "$FILES_JSON" \ + '{source_repo:$repo, source_sha:$sha, vendored_at:$at, files:$files}' \ + > "$DEST/_vendor.json"; then + echo "ERROR: provenance stamp write failed" >&2; exit 1 +fi + +echo "Vendored $(echo "$MANIFEST" | wc -w | tr -d ' ') engine scripts to $DEST (source $SHA)" diff --git a/skills/automation-health-dashboard/SKILL.md b/skills/automation-health-dashboard/SKILL.md index f444e80..52729b4 100644 --- a/skills/automation-health-dashboard/SKILL.md +++ b/skills/automation-health-dashboard/SKILL.md @@ -15,7 +15,7 @@ Generate a unified executive-facing dashboard combining link-health and dep-bump ## Prerequisites -- `bash` 4+ (macOS ships 3.2; use `brew install bash` for 4+) +- `bash` 3.2+ (works with the macOS system default; no bash-4 features used) - `gh` (GitHub CLI, authenticated with org access) - `jq` (JSON processor) diff --git a/skills/dep-bump-fixer/SKILL.md b/skills/dep-bump-fixer/SKILL.md index 042a7d8..484d10e 100644 --- a/skills/dep-bump-fixer/SKILL.md +++ b/skills/dep-bump-fixer/SKILL.md @@ -16,7 +16,7 @@ Respond to scanner-created issues (`[dep-bump]` title prefix) with severity-appr ## Prerequisites -- `bash` 4+ (macOS ships 3.2; use `brew install bash` for 4+) +- `bash` 3.2+ (works with the macOS system default; no bash-4 features used) - `gh` (GitHub CLI, authenticated with org access) - `jq` (JSON processor) diff --git a/skills/dep-bump-scanner/SKILL.md b/skills/dep-bump-scanner/SKILL.md index aabe544..ba34650 100644 --- a/skills/dep-bump-scanner/SKILL.md +++ b/skills/dep-bump-scanner/SKILL.md @@ -16,7 +16,7 @@ Monitor all repositories in a GitHub organization for open Dependabot PRs. Class ## Prerequisites -- `bash` 4+ (macOS ships 3.2; use `brew install bash` for 4+) +- `bash` 3.2+ (works with the macOS system default; no bash-4 features used) - `gh` (GitHub CLI, authenticated with org access) - `jq` (JSON processor) diff --git a/skills/link-health-fixer/SKILL.md b/skills/link-health-fixer/SKILL.md index e3586b7..6291588 100644 --- a/skills/link-health-fixer/SKILL.md +++ b/skills/link-health-fixer/SKILL.md @@ -16,7 +16,7 @@ Re-verify and fix broken links reported by the link-health-scanner. Creates fork ## Prerequisites -- `bash` 4+ (macOS ships 3.2; use `brew install bash` for 4+) +- `bash` 3.2+ (works with the macOS system default; no bash-4 features used) - `gh` (GitHub CLI, authenticated with fork/PR permissions) - `jq` (JSON processor) - `curl` (for external link verification and Wayback Machine) diff --git a/skills/link-health-scanner/SKILL.md b/skills/link-health-scanner/SKILL.md index 59e41fc..7cb1537 100644 --- a/skills/link-health-scanner/SKILL.md +++ b/skills/link-health-scanner/SKILL.md @@ -16,7 +16,7 @@ Scan all repositories in a GitHub organization for broken links. Creates GitHub ## Prerequisites -- `bash` 4+ (macOS ships 3.2; use `brew install bash` for 4+) +- `bash` 3.2+ (works with the macOS system default; no bash-4 features used) - `lychee` >= 0.23.0 (link checker: `brew install lychee`) — the scanner relies on lychee's default exclusion of URLs inside code blocks; do not pass `--include-verbatim` - `gh` (GitHub CLI, authenticated with org access) diff --git a/skills/repoman/SKILL.md b/skills/repoman/SKILL.md index 5672c69..2ebf2b5 100644 --- a/skills/repoman/SKILL.md +++ b/skills/repoman/SKILL.md @@ -42,18 +42,24 @@ to infer state. You write it only by calling the setup writer. ## Where the scripts live -- **Bundled with this skill** (`scripts/org.sh`): the enrolled-set resolver and +The entire engine is **bundled with this plugin** under +`${CLAUDE_PLUGIN_ROOT}/skills/repoman/scripts/` — there is no separate automation +checkout to install or locate. Resolve that directory at runtime via +`${CLAUDE_PLUGIN_ROOT}` and point the orchestrator at it with +`REPOMAN_RUN_BIN="${CLAUDE_PLUGIN_ROOT}/skills/repoman/scripts"` (the existing +run-time override `repoman-run.sh` reads). The scripts are a flat set, so each one +sources its siblings (`program-lib.sh`, `org.sh`, …) from that same directory. + +- **Read-state / reconcile helpers** (`org.sh`): the enrolled-set resolver and `repoman_match_name` (bare-name → enrolled owner lookup). Source these to read state and reconcile names. -- **In an automation checkout** (not bundled): the writer `repoman-setup.sh`, - the capability gate `repoman-check.sh`, and the requirements parser - `repoman-requirements.sh`. Run these from an automation checkout. - `repoman-setup.sh` and `repoman-check.sh` are **commands** — run them with - `--help` first to see full usage. `repoman-requirements.sh` is **not** a - command: it is a sourced library that exposes the function +- **The writer `repoman-setup.sh`, the capability gate `repoman-check.sh`, and + the orchestrator `repoman-run.sh`** are **commands** — run them from the bundled + scripts dir, `--help` first to see full usage. `repoman-requirements.sh` is + **not** a command: it is a sourced library that exposes the function `repoman_parse_requirements`. You must **source it, then call the function** — - `source /scripts/repoman-requirements.sh` then - `repoman_parse_requirements `. Never execute it directly (e.g. + `source "${CLAUDE_PLUGIN_ROOT}/skills/repoman/scripts/repoman-requirements.sh"` + then `repoman_parse_requirements `. Never execute it directly (e.g. `repoman-requirements.sh `): a direct run does **not** emit the JSON — once rossoctl/automation#113 lands it fails loud with this same guidance, and before that it exits 0 with empty output (which, captured and parsed as @@ -255,7 +261,7 @@ any other reason, re-prompt with the specific reason the validator gave — do 1. locate that program's `SKILL.md` (via the program→skill-dir map above, e.g. `skills/github-pr-review/SKILL.md`); 2. **source** the parser library once, then call the function: - `source /scripts/repoman-requirements.sh` → + `source "${CLAUDE_PLUGIN_ROOT}/skills/repoman/scripts/repoman-requirements.sh"` → `repoman_parse_requirements ` → `{"pat_scopes":[...],"labels_required":[...],"labels_applied":[...],"programs":[...]}`. Do **not** execute the script directly — it is a sourced library, not a @@ -376,8 +382,10 @@ This keeps the setup/activate/reconcile flows fully usable on their own. 4 features) - `gh` (GitHub CLI, authenticated with org access) — for org detection/expansion - `jq` (JSON processor) -- An **automation checkout** on `PATH` or at a known location, for - `repoman-setup.sh`, `repoman-check.sh`, and `repoman-requirements.sh`. +- The RepoMan **engine**, bundled with this plugin at + `${CLAUDE_PLUGIN_ROOT}/skills/repoman/scripts/` (`repoman-setup.sh`, + `repoman-check.sh`, `repoman-run.sh`, `repoman-requirements.sh`, and the + program scanners). No separate checkout is required. ### Requirements (machine-readable) diff --git a/skills/repoman/scripts/_vendor.json b/skills/repoman/scripts/_vendor.json new file mode 100644 index 0000000..5ada4e7 --- /dev/null +++ b/skills/repoman/scripts/_vendor.json @@ -0,0 +1,30 @@ +{ + "source_repo": "rossoctl/automation", + "source_sha": "0311752eba5350c74b8c9650137d3e6da65bd547", + "vendored_at": "2026-10-23T03:46:40Z", + "files": [ + "automation-health-dashboard.sh", + "core.sh", + "dep-bump-fixer.sh", + "dep-bump-scanner.sh", + "extract-broken-links.sh", + "fork.sh", + "github-api.sh", + "link-health-fixer.sh", + "link-health-scanner.sh", + "org.sh", + "parse-diff-map.sh", + "pr-review-fixer.sh", + "pr-review-impact.sh", + "pr-review-lib.sh", + "pr-review-scanner.sh", + "program-lib.sh", + "repo-sync.sh", + "repoman-check.sh", + "repoman-index.sh", + "repoman-requirements.sh", + "repoman-run.sh", + "repoman-setup.sh", + "weekly-report.sh" + ] +} diff --git a/skills/repoman/scripts/automation-health-dashboard.sh b/skills/repoman/scripts/automation-health-dashboard.sh new file mode 100644 index 0000000..2887e0d --- /dev/null +++ b/skills/repoman/scripts/automation-health-dashboard.sh @@ -0,0 +1,624 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Automation Health Dashboard Generator +# Combines link-health and dep-bump program metrics into a single executive- +# facing markdown dashboard. Pushes to a standing fork-based PR. +# +# Usage: +# bash automation-health-dashboard.sh --help +# bash automation-health-dashboard.sh --dry-run +# bash automation-health-dashboard.sh --live +# ============================================================================= + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- CLI args --- +DRY_RUN=true +VERBOSE=false +SHOW_HELP=false +MAIN_REPO_DIR="${MAIN_REPO_DIR:-}" +INDEX_FILE_ARG="" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) DRY_RUN=true; shift ;; + --live) DRY_RUN=false; shift ;; + --reports-dir) REPORTS_DIR="$2"; shift 2 ;; + --index) INDEX_FILE_ARG="$2"; shift 2 ;; + --main-repo-dir) MAIN_REPO_DIR="$2"; shift 2 ;; + --verbose) VERBOSE=true; shift ;; + --help|-h) SHOW_HELP=true; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if [ "$SHOW_HELP" = true ]; then + cat <<'HELP' +Automation Health Dashboard Generator + +Combines link-health and dep-bump program metrics into a single executive- +facing markdown file. Pushes to a standing fork-based PR in the org's main repo. + +Usage: + bash automation-health-dashboard.sh [OPTIONS] + +Options: + --dry-run Generate and preview dashboard (default) + --live Commit and push to fork, create/update PR + --reports-dir DIR Base reports directory (default: $REPORTS_DIR or ./reports) + --index PATH Path to _index.json (default: $REPOMAN_INDEX_FILE or + /_index.json). When present, drives which + programs render and their headings; falls back to + disk-derived discovery when absent. + --main-repo-dir DIR Path to the report-target repo clone, overriding the + REPOS_DIR-derived default (default: $MAIN_REPO_DIR) + --verbose Print diagnostic output + --help, -h Show this help + +Environment: + REPORTS_DIR Base directory containing program report subdirs + REPOMAN_INDEX_FILE Path to _index.json (default: /_index.json) + MAIN_REPO_DIR Path to the report-target repo clone (live mode git ops); + overrides the REPOS_DIR-derived default + FORK_OWNER Fork owner for cross-fork PRs +HELP + exit 0 +fi + +# Resolve deployment constants (repos_dir, fork_owner) from ~/.repoman/config.json. +repoman_config + +# Report-PR destination. The org main repo's docs/ folder feeds the docs site +# (rossoctl.dev) and cannot host machine-generated reports, so the standing +# dashboard PR lands under automation-health/ in the automation repo. A single +# file, overwritten in place each run: trend tooling reconstructs history by +# replaying git commit parents, so we store state (not dated snapshots) and +# avoid the files-vs-diffs-on-Git anti-pattern (rossoctl/automation#44). +# TODO(RepoMan Phase 2): move report_target_repo/source_repo to programs/health-dashboard.json. +REPORT_TARGET_REPO="rossoctl/automation" +SOURCE_REPO="rossoctl/automation" +REPORT_TARGET_OWNER="${REPORT_TARGET_REPO%%/*}" +REPORT_TARGET_NAME="${REPORT_TARGET_REPO##*/}" +REPORT_TARGET_PATH="automation-health/automation-health.md" +# Clone dir for the report target: honor an explicit --main-repo-dir/MAIN_REPO_DIR +# override, else derive from the owner-namespaced layout ($REPOS_DIR//). +REPORT_TARGET_DIR="${MAIN_REPO_DIR:-$REPOS_DIR/$REPORT_TARGET_OWNER/$REPORT_TARGET_NAME}" + +# --- Validate inputs --- +if [ -z "${REPORTS_DIR:-}" ]; then + # Default to ~/reports -- the parent the scanners/fixers now write their + # per-program dirs (and the shared _index.json) under. Fall back to ./reports + # for a repo-local layout if ~/reports does not exist yet. + if [ -d "$HOME/reports" ]; then + REPORTS_DIR="$HOME/reports" + elif [ -d "./reports" ]; then + REPORTS_DIR="./reports" + else + echo "ERROR: REPORTS_DIR is not set and neither ~/reports nor ./reports exists." + echo "Export it to the directory containing program report subdirs:" + echo " export REPORTS_DIR=~/reports" + exit 1 + fi +fi + +INDEX_FILE="${INDEX_FILE_ARG:-${REPOMAN_INDEX_FILE:-$REPORTS_DIR/_index.json}}" + +# Program-id -> display-name lookup (bash 3.2 safe, no assoc arrays). Only ids +# listed here have a corresponding extraction block further down; an index +# entry for any other id is a known/skippable program, not an error. +display_name_for() { + case "$1" in + link-health) echo "Link Health" ;; + dep-bump) echo "Dependency Bumps" ;; + *) echo "" ;; + esac +} + +LINK_SCAN_DIR="" +DEP_BUMP_DIR="" +HAS_LINK_HEALTH=false +HAS_DEP_BUMP=false +LINK_HEALTH_HEADING="Link Health" +DEP_BUMP_HEADING="Dependency Bumps" + +if [ -f "$INDEX_FILE" ]; then + if ! jq empty "$INDEX_FILE" >/dev/null 2>&1; then + echo "ERROR: Index file is not valid JSON: $INDEX_FILE" + exit 1 + fi + + while IFS= read -r prog_id; do + [ -z "$prog_id" ] && continue + known_heading=$(display_name_for "$prog_id") + if [ -z "$known_heading" ]; then + echo "Skipping unknown program id from index (no extraction block): $prog_id" + continue + fi + + entry_path=$(jq -r --arg id "$prog_id" '.[$id].report_path // ""' "$INDEX_FILE") + entry_display=$(jq -r --arg id "$prog_id" '.[$id].display_name // ""' "$INDEX_FILE") + [ -z "$entry_display" ] && entry_display="$known_heading" + + if [ -z "$entry_path" ] || [ ! -f "$entry_path/latest.json" ] || [ ! -f "$entry_path/history.json" ]; then + echo "Skipping program with missing reports: $prog_id ($entry_path)" + continue + fi + + case "$prog_id" in + link-health) + LINK_SCAN_DIR="$entry_path" + HAS_LINK_HEALTH=true + LINK_HEALTH_HEADING="$entry_display" + ;; + dep-bump) + DEP_BUMP_DIR="$entry_path" + HAS_DEP_BUMP=true + DEP_BUMP_HEADING="$entry_display" + ;; + esac + done < <(jq -r 'keys[]' "$INDEX_FILE") +else + # Disk-derived fallback: no index, probe the known report subdirs directly. + LINK_SCAN_DIR="$REPORTS_DIR/link-health" + DEP_BUMP_DIR="$REPORTS_DIR/dep-bump" + + if [ -f "$LINK_SCAN_DIR/latest.json" ] && [ -f "$LINK_SCAN_DIR/history.json" ]; then + HAS_LINK_HEALTH=true + LINK_HEALTH_HEADING=$(display_name_for "link-health") + fi + if [ -f "$DEP_BUMP_DIR/latest.json" ] && [ -f "$DEP_BUMP_DIR/history.json" ]; then + HAS_DEP_BUMP=true + DEP_BUMP_HEADING=$(display_name_for "dep-bump") + fi +fi + +if [ "$HAS_LINK_HEALTH" = false ] && [ "$HAS_DEP_BUMP" = false ]; then + echo "ERROR: No program reports found in $REPORTS_DIR" + echo "Expected: $REPORTS_DIR/link-health/latest.json and/or $REPORTS_DIR/dep-bump/latest.json" + exit 1 +fi + +PROGRAMS_ACTIVE=0 +if [ "$HAS_LINK_HEALTH" = true ]; then PROGRAMS_ACTIVE=$((PROGRAMS_ACTIVE + 1)); fi +if [ "$HAS_DEP_BUMP" = true ]; then PROGRAMS_ACTIVE=$((PROGRAMS_ACTIVE + 1)); fi + +# --- Setup --- +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT + +SCAN_TIME_ET=$(TZ="America/New_York" date +"%Y-%m-%d %H:%M ET") + +echo "=== Automation Health Dashboard ===" +echo "Reports dir: $REPORTS_DIR" +echo "Programs available: $PROGRAMS_ACTIVE" +if [ "$DRY_RUN" = true ]; then + echo "Mode: DRY RUN" +else + echo "Mode: LIVE" +fi +echo "" + +# ============================================================================= +# Step 2: Compute executive summary metrics +# ============================================================================= + +LH_ISSUES_CREATED=0 +LH_ISSUES_CLOSED=0 +DB_ISSUES_CREATED=0 +DB_ISSUES_CLOSED=0 +DB_FIXER_CLOSED=0 +TOTAL_PRS_OPENED=0 +LAST_SCAN_DATE="unknown" + +if [ "$HAS_LINK_HEALTH" = true ]; then + LH_ISSUES_CREATED=$(jq '[.[] | .issues_created // 0] | add // 0' "$LINK_SCAN_DIR/history.json") + LH_ISSUES_CLOSED=$(jq '[.[] | .issues_closed // 0] | add // 0' "$LINK_SCAN_DIR/history.json") + lh_last_date=$(jq -r '.date // ""' "$LINK_SCAN_DIR/latest.json") + if [ -n "$lh_last_date" ]; then + LAST_SCAN_DATE="${lh_last_date%%T*}" + fi +fi + +if [ "$HAS_DEP_BUMP" = true ]; then + DB_ISSUES_CREATED=$(jq '[.[] | .issues_created // 0] | add // 0' "$DEP_BUMP_DIR/history.json") + DB_ISSUES_CLOSED=$(jq '[.[] | .issues_closed // 0] | add // 0' "$DEP_BUMP_DIR/history.json") + db_last_date=$(jq -r '.date // ""' "$DEP_BUMP_DIR/latest.json") + if [ -n "$db_last_date" ]; then + db_date="${db_last_date%%T*}" + if [ "$LAST_SCAN_DATE" = "unknown" ] || [[ "$db_date" > "$LAST_SCAN_DATE" ]]; then + LAST_SCAN_DATE="$db_date" + fi + fi + + if [ -f "$DEP_BUMP_DIR/fixer-latest.json" ]; then + DB_FIXER_CLOSED=$(jq '.issues_closed // 0' "$DEP_BUMP_DIR/fixer-latest.json") + fixer_config_prs=$(jq '.config_prs_created // 0' "$DEP_BUMP_DIR/fixer-latest.json") + TOTAL_PRS_OPENED=$((TOTAL_PRS_OPENED + fixer_config_prs)) + fi +fi + +TOTAL_ISSUES_CREATED=$((LH_ISSUES_CREATED + DB_ISSUES_CREATED)) +TOTAL_ISSUES_RESOLVED=$((LH_ISSUES_CLOSED + DB_ISSUES_CLOSED + DB_FIXER_CLOSED)) +HOURS_SAVED=$(echo "$TOTAL_ISSUES_RESOLVED" | awk '{printf "%.1f", $1 * 0.25}') + +if [ "$VERBOSE" = true ]; then + echo "Executive metrics:" + echo " Issues created: $TOTAL_ISSUES_CREATED (LH: $LH_ISSUES_CREATED, DB: $DB_ISSUES_CREATED)" + echo " Issues resolved: $TOTAL_ISSUES_RESOLVED (LH: $LH_ISSUES_CLOSED, DB: $DB_ISSUES_CLOSED, fixer: $DB_FIXER_CLOSED)" + echo " PRs opened: $TOTAL_PRS_OPENED" + echo " Hours saved: $HOURS_SAVED" + echo "" +fi + +# ============================================================================= +# Step 3: Compute link-health section +# ============================================================================= + +LH_REPOS_SCANNED=0 +LH_TOTAL_LINKS=0 +LH_BROKEN_INTERNAL=0 +LH_BROKEN_EXTERNAL=0 +LH_FIRST_INTERNAL=0 +LH_FIRST_EXTERNAL=0 +LH_TREND_TABLE="| - | - | - | - |" + +if [ "$HAS_LINK_HEALTH" = true ]; then + LH_REPOS_SCANNED=$(jq '.repos_scanned // 0' "$LINK_SCAN_DIR/latest.json") + LH_TOTAL_LINKS=$(jq '.total_links_checked // 0' "$LINK_SCAN_DIR/latest.json") + LH_BROKEN_INTERNAL=$(jq '[.broken[] | select(.category == "internal")] | length' "$LINK_SCAN_DIR/latest.json" 2>/dev/null || echo "0") + LH_BROKEN_EXTERNAL=$(jq '[.broken[] | select(.category == "external")] | length' "$LINK_SCAN_DIR/latest.json" 2>/dev/null || echo "0") + + # First scan values for trend comparison + LH_FIRST_INTERNAL=$(jq '.[0].broken_internal // 0' "$LINK_SCAN_DIR/history.json") + LH_FIRST_EXTERNAL=$(jq '.[0].broken_external // 0' "$LINK_SCAN_DIR/history.json") + + LH_INTERNAL_DELTA=$((LH_BROKEN_INTERNAL - LH_FIRST_INTERNAL)) + LH_EXTERNAL_DELTA=$((LH_BROKEN_EXTERNAL - LH_FIRST_EXTERNAL)) + + # Format deltas with sign + lh_int_trend="" + if [ "$LH_INTERNAL_DELTA" -gt 0 ]; then lh_int_trend="+$LH_INTERNAL_DELTA from first scan" + elif [ "$LH_INTERNAL_DELTA" -lt 0 ]; then lh_int_trend="$LH_INTERNAL_DELTA from first scan" + fi + lh_ext_trend="" + if [ "$LH_EXTERNAL_DELTA" -gt 0 ]; then lh_ext_trend="+$LH_EXTERNAL_DELTA from first scan" + elif [ "$LH_EXTERNAL_DELTA" -lt 0 ]; then lh_ext_trend="$LH_EXTERNAL_DELTA from first scan" + fi + + # Trend table (last 10 scans) + LH_TREND_TABLE=$(jq -r ' + .[-10:] | reverse | .[] | + "| \(.date | split("T")[0]) | \(.broken_internal) | \(.broken_external) | \(if .new > .fixed then "+\(.new - .fixed)" elif .new < .fixed then "\(.new - .fixed)" else "0" end) |" + ' "$LINK_SCAN_DIR/history.json" 2>/dev/null || echo "| - | - | - | - |") +fi + +# ============================================================================= +# Step 4: Compute dep-bump section +# ============================================================================= + +DB_REPOS_SCANNED=0 +DB_TOTAL_OPEN_PRS=0 +DB_STALE_COUNT=0 +DB_SLA_COMPLIANCE=0 +DB_MEDIAN_TTM=0 +DB_BASELINE_TTM=0 +DB_COVERAGE_REPOS=0 +DB_COVERAGE_TOTAL=0 +DB_COVERAGE_PCT=0 +DB_TIER_TABLE="| - | - | - | - |" +DB_TREND_TABLE="| - | - | - | - |" + +if [ "$HAS_DEP_BUMP" = true ]; then + DB_REPOS_SCANNED=$(jq '.repos_scanned // 0' "$DEP_BUMP_DIR/latest.json") + DB_TOTAL_OPEN_PRS=$(jq '.total_open_prs // 0' "$DEP_BUMP_DIR/latest.json") + DB_STALE_COUNT=$(jq '.stale_prs | length' "$DEP_BUMP_DIR/latest.json" 2>/dev/null || echo "0") + DB_COVERAGE_REPOS=$(jq '.repos_with_dependabot // 0' "$DEP_BUMP_DIR/latest.json") + DB_COVERAGE_TOTAL=$DB_REPOS_SCANNED + + if [ "$DB_COVERAGE_TOTAL" -gt 0 ]; then + DB_COVERAGE_PCT=$((DB_COVERAGE_REPOS * 100 / DB_COVERAGE_TOTAL)) + fi + + if [ "$DB_TOTAL_OPEN_PRS" -gt 0 ]; then + DB_SLA_COMPLIANCE=$(( (DB_TOTAL_OPEN_PRS - DB_STALE_COUNT) * 100 / DB_TOTAL_OPEN_PRS )) + else + DB_SLA_COMPLIANCE=100 + fi + + if [ -f "$DEP_BUMP_DIR/fixer-latest.json" ]; then + DB_MEDIAN_TTM=$(jq '.metrics.median_time_to_merge // 0' "$DEP_BUMP_DIR/fixer-latest.json") + fi + if [ -f "$DEP_BUMP_DIR/baseline.json" ]; then + DB_BASELINE_TTM=$(jq '.median_time_to_merge_days // 0' "$DEP_BUMP_DIR/baseline.json") + fi + + # Tier breakdown from stale_prs array + DB_TIER_TABLE=$(jq -r ' + .stale_prs as $all | + .total_open_prs as $total | + [ + {tier: "Critical", sla: "3d", + stale: ([$all[] | select(.severity == "critical")] | length)}, + {tier: "High", sla: "7d", + stale: ([$all[] | select(.severity == "high")] | length)}, + {tier: "Medium", sla: "30d", + stale: ([$all[] | select(.severity == "medium")] | length)}, + {tier: "Routine", sla: "14d", + stale: ([$all[] | select(.severity == "routine")] | length)}, + {tier: "Major", sla: "30d", + stale: ([$all[] | select(.severity == "major")] | length)} + ] | .[] | + "| \(.tier) | \(.stale) | \(.sla) |" + ' "$DEP_BUMP_DIR/latest.json" 2>/dev/null || echo "| - | - | - |") + + # Trend table (last 10 scans) + DB_TREND_TABLE=$(jq -r ' + .[-10:] | reverse | .[] | + "| \(.date | split("T")[0]) | \(.stale_security // 0) | \(.stale_routine // 0) | \(if .new > .fixed then "+\(.new - .fixed)" elif .new < .fixed then "\(.new - .fixed)" else "0" end) |" + ' "$DEP_BUMP_DIR/history.json" 2>/dev/null || echo "| - | - | - | - |") +fi + +# ============================================================================= +# Step 5: Compute cross-program coverage +# ============================================================================= + +COVERAGE_TABLE="" +REPOS_UNDER_ONE=0 +REPOS_UNDER_ALL=0 +TOTAL_UNIQUE_REPOS=0 + +# Collect repos from each program +lh_repos="" +db_repos="" + +if [ "$HAS_LINK_HEALTH" = true ]; then + # .repo is the full owner/name ref (extract-broken-links.sh emits it verbatim). + # Key on the full ref so it matches db_repos below -- stripping the owner here + # would make the two program sets un-mergeable (no repo could match both). + lh_repos=$(jq -r '[.broken[].repo] | unique | .[]' "$LINK_SCAN_DIR/latest.json" 2>/dev/null | sort -u || true) + # Also include repos scanned (from history, repos_scanned is a count not a list) + # Fall back to broken repos as proxy for "scanned repos" +fi + +if [ "$HAS_DEP_BUMP" = true ]; then + # Repos with dependabot activity -- .repo is the full owner/name ref, same + # shape as lh_repos, so the sort -u merge and grep -qxF cross-check align. + db_repos=$(jq -r '([.stale_prs[].repo] + [.coverage_gaps[].repo]) | unique | .[]' "$DEP_BUMP_DIR/latest.json" 2>/dev/null | sort -u || true) +fi + +# Merge and produce table +all_repos=$(printf '%s\n%s\n' "$lh_repos" "$db_repos" | sort -u | grep -v '^$' || true) +# Count non-empty lines. `grep -c` already prints 0 on no match AND returns 1, +# so a `|| echo 0` fallback would append a SECOND "0" -- yielding a two-line +# value that breaks the integer test at the coverage-percent guard below. Guard +# the empty case explicitly instead of relying on grep's rc. +if [ -n "$all_repos" ]; then + TOTAL_UNIQUE_REPOS=$(printf '%s\n' "$all_repos" | grep -c .) +else + TOTAL_UNIQUE_REPOS=0 +fi + +while IFS= read -r repo; do + [ -z "$repo" ] && continue + has_lh="no" + has_db="no" + count=0 + + if echo "$lh_repos" | grep -qxF "$repo"; then + has_lh="yes" + count=$((count + 1)) + fi + if echo "$db_repos" | grep -qxF "$repo"; then + has_db="yes" + count=$((count + 1)) + fi + + COVERAGE_TABLE="${COVERAGE_TABLE}| $repo | $has_lh | $has_db | $count | +" + if [ "$count" -ge 1 ]; then REPOS_UNDER_ONE=$((REPOS_UNDER_ONE + 1)); fi + if [ "$count" -ge "$PROGRAMS_ACTIVE" ]; then REPOS_UNDER_ALL=$((REPOS_UNDER_ALL + 1)); fi +done <<< "$all_repos" + +COVERAGE_ONE_PCT=0 +COVERAGE_ALL_PCT=0 +if [ "$TOTAL_UNIQUE_REPOS" -gt 0 ]; then + COVERAGE_ONE_PCT=$((REPOS_UNDER_ONE * 100 / TOTAL_UNIQUE_REPOS)) + COVERAGE_ALL_PCT=$((REPOS_UNDER_ALL * 100 / TOTAL_UNIQUE_REPOS)) +fi + +# ============================================================================= +# Step 6: Cron health +# ============================================================================= + +# Static entries — future version should read from jobs.json +CRON_TABLE="| link-health-scanner | Mon/Wed/Fri 7am ET | $LAST_SCAN_DATE | ok | +| link-health-fixer | Tue/Thu 8am ET | $LAST_SCAN_DATE | ok | +| dep-bump-scanner | Tue/Thu 10am ET | $LAST_SCAN_DATE | ok | +| dep-bump-fixer | Tue/Thu 12pm ET | $LAST_SCAN_DATE | ok |" + +# ============================================================================= +# Step 7: Generate markdown +# ============================================================================= + +# Base: header + Executive Summary (always rendered). +cat > "$TMPDIR/automation-health.md" << DASHBOARD_EOF +# Automation Health Dashboard + +> Last updated: $SCAN_TIME_ET | Programs: $PROGRAMS_ACTIVE active + +## Executive Summary + +| Metric | Value | +|--------|-------| +| Total issues auto-created | $TOTAL_ISSUES_CREATED (link-health: $LH_ISSUES_CREATED, dep-bump: $DB_ISSUES_CREATED) | +| Total issues auto-resolved | $TOTAL_ISSUES_RESOLVED | +| Total PRs auto-opened | $TOTAL_PRS_OPENED | +| Estimated hours saved | ${HOURS_SAVED} hrs (at 15 min/resolved issue) | +| Programs active | $PROGRAMS_ACTIVE | +| Last successful scan | $LAST_SCAN_DATE | +DASHBOARD_EOF + +# Per-program sections render only when discovery found that program, so an +# index with only one program does not emit the other's heading (empty section). +if [ "$HAS_LINK_HEALTH" = true ]; then + cat >> "$TMPDIR/automation-health.md" << DASHBOARD_EOF + +## $LINK_HEALTH_HEADING + +| Metric | Value | Trend | +|--------|-------|-------| +| Repos scanned | $LH_REPOS_SCANNED | | +| Total links checked | $LH_TOTAL_LINKS | | +| Broken (internal) | $LH_BROKEN_INTERNAL | $lh_int_trend | +| Broken (external) | $LH_BROKEN_EXTERNAL | $lh_ext_trend | +| Issues created (cumulative) | $LH_ISSUES_CREATED | | +| Issues resolved (cumulative) | $LH_ISSUES_CLOSED | | + +### Trend (last 10 scans) + +| Date | Internal | External | Delta | +|------|----------|----------|-------| +$LH_TREND_TABLE +DASHBOARD_EOF +fi + +if [ "$HAS_DEP_BUMP" = true ]; then + cat >> "$TMPDIR/automation-health.md" << DASHBOARD_EOF + +## $DEP_BUMP_HEADING + +| Metric | Value | Trend | +|--------|-------|-------| +| Repos scanned | $DB_REPOS_SCANNED | | +| Open Dependabot PRs | $DB_TOTAL_OPEN_PRS | | +| Stale PRs (SLA breached) | $DB_STALE_COUNT | baseline: $(jq '.stale_count_at_baseline // "N/A"' "$DEP_BUMP_DIR/baseline.json" 2>/dev/null || echo "N/A") | +| SLA compliance rate | ${DB_SLA_COMPLIANCE}% | | +| Median time-to-merge | ${DB_MEDIAN_TTM}d | baseline: ${DB_BASELINE_TTM}d | +| Dependabot coverage | ${DB_COVERAGE_PCT}% ($DB_COVERAGE_REPOS/$DB_COVERAGE_TOTAL repos) | | + +### By Severity Tier + +| Tier | Stale | SLA | +|------|-------|-----| +$DB_TIER_TABLE + +### Patch Velocity (last 10 scans) + +| Date | Stale Security | Stale Routine | Delta | +|------|----------------|---------------|-------| +$DB_TREND_TABLE +DASHBOARD_EOF +fi + +# Tail: cross-cutting sections (always rendered). +cat >> "$TMPDIR/automation-health.md" << DASHBOARD_EOF + +## Cross-Program Coverage + +| Repo | Link Health | Dep Bump | Programs | +|------|-------------|----------|----------| +$COVERAGE_TABLE + +### Coverage Summary +- Repos under at least one program: $REPOS_UNDER_ONE / $TOTAL_UNIQUE_REPOS (${COVERAGE_ONE_PCT}%) +- Repos under all programs: $REPOS_UNDER_ALL / $TOTAL_UNIQUE_REPOS (${COVERAGE_ALL_PCT}%) + +## Cron Health + +| Job | Schedule | Last Run | Status | +|-----|----------|----------|--------| +$CRON_TABLE + +--- +*Generated by RepoMan Automation Health Dashboard. Do not edit manually.* +DASHBOARD_EOF + +echo "Dashboard generated ($TMPDIR/automation-health.md)" + +# ============================================================================= +# Step 8: Output or commit +# ============================================================================= + +if [ "$DRY_RUN" = true ]; then + echo "" + echo "[DRY RUN] Dashboard preview:" + echo "---" + cat "$TMPDIR/automation-health.md" + echo "---" + echo "[DRY RUN] Would push $REPORT_TARGET_PATH to fork and create/update PR against $REPORT_TARGET_REPO" +else + if [ ! -d "$REPORT_TARGET_DIR/.git" ]; then + echo "ERROR: $REPORT_TARGET_DIR does not appear to be a git repository." + echo "Export MAIN_REPO_DIR or set REPOS_DIR so $REPORT_TARGET_REPO can be found:" + echo " export MAIN_REPO_DIR=$REPOS_DIR/$REPORT_TARGET_OWNER/$REPORT_TARGET_NAME" + exit 1 + fi + + FORK_REMOTE="$FORK_OWNER" + DASHBOARD_BRANCH="automation/health-dashboard" + + cd "$REPORT_TARGET_DIR" + + # Ensure the fork remote exists AND points at the current target. set-url + # corrects a stale remote (e.g. one left by a prior deployment pointing at the + # old report repo); the || add branch handles the not-yet-registered case. + fork_url="https://github.com/$FORK_OWNER/${REPORT_TARGET_NAME}.git" + git remote set-url "$FORK_REMOTE" "$fork_url" 2>/dev/null \ + || git remote add "$FORK_REMOTE" "$fork_url" + + # Fetch fork's branch if it exists, otherwise create from main + if git fetch "$FORK_REMOTE" "$DASHBOARD_BRANCH" 2>/dev/null; then + git checkout -B "$DASHBOARD_BRANCH" "$FORK_REMOTE/$DASHBOARD_BRANCH" + else + git fetch "$FORK_REMOTE" main 2>/dev/null || true + git checkout -B "$DASHBOARD_BRANCH" "$FORK_REMOTE/main" 2>/dev/null \ + || git checkout -B "$DASHBOARD_BRANCH" + fi + + mkdir -p "$(dirname "$REPORT_TARGET_PATH")" + cp "$TMPDIR/automation-health.md" "$REPORT_TARGET_PATH" + git add "$REPORT_TARGET_PATH" + git commit -s -m "docs: Update automation health dashboard ($SCAN_TIME_ET)" 2>/dev/null || echo "No changes to commit" + git push "$FORK_REMOTE" "$DASHBOARD_BRANCH" 2>/dev/null || echo "WARN: Failed to push dashboard to fork" + + # Create or update standing cross-fork PR + existing_pr=$(gh api "repos/$REPORT_TARGET_REPO/pulls?head=$FORK_OWNER:$DASHBOARD_BRANCH&state=open" \ + --jq '.[0].number' 2>/dev/null || echo "") + + pr_body="## Summary + +Auto-updated by RepoMan Automation Health Dashboard. This PR is continuously updated with each generation. Merge when convenient. + +| Metric | Value | +|--------|-------| +| Programs active | $PROGRAMS_ACTIVE | +| Issues created (cumulative) | $TOTAL_ISSUES_CREATED | +| Issues resolved (cumulative) | $TOTAL_ISSUES_RESOLVED | +| Estimated hours saved | ${HOURS_SAVED} hrs | + +## Related issue(s) + +- $REPORT_TARGET_REPO#1260 + +## Automation program + +Generated by the [RepoMan Automation Health Dashboard](https://github.com/$SOURCE_REPO/blob/main/standing-orders/health-dashboard.md)." + + if [ -z "$existing_pr" ] || [ "$existing_pr" = "null" ]; then + gh pr create --repo "$REPORT_TARGET_REPO" \ + --head "$FORK_OWNER:$DASHBOARD_BRANCH" --base main \ + --title "docs: Automation health dashboard (auto-updated)" \ + --body "$pr_body" 2>/dev/null || echo "WARN: Failed to create dashboard PR" + else + gh pr edit "$existing_pr" --repo "$REPORT_TARGET_REPO" --body "$pr_body" 2>/dev/null || true + fi + + echo "Dashboard committed and pushed" +fi + +echo "" +echo "=== Done ===" diff --git a/skills/repoman/scripts/core.sh b/skills/repoman/scripts/core.sh new file mode 100644 index 0000000..1616afc --- /dev/null +++ b/skills/repoman/scripts/core.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# Core utilities for the automation programs: workspace/temp management, portable +# date math, JSON-schema validation, scan diffing, and report file I/O. +# +# ## Portability +# Targets bash 3.2+ (macOS default) through modern bash. No associative arrays, +# no mapfile/readarray, no GNU-only sed/grep/date flags without a BSD fallback. +# +# Load-once guard: safe to source multiple times (self-sourcing modules may pull +# this in more than once). +[ -n "${_CORE_SH_LOADED:-}" ] && return +_CORE_SH_LOADED=1 + +# ============================================================================= +# WORKSPACE MANAGEMENT +# ============================================================================= + +# Create a temp directory and register automatic cleanup on script exit. +# After calling this, use $PROGRAM_TMPDIR for all temp files. +# +# Usage: setup_workspace "link-fixer" +# Args: +# $1 - prefix for the temp directory name (default: "program") +setup_workspace() { + local prefix="${1:-program}" + + PROGRAM_TMPDIR=$(mktemp -d "/tmp/${prefix}-XXXXXX") + # Automatic cleanup: remove the temp dir when the script exits + trap 'rm -rf "$PROGRAM_TMPDIR"' EXIT +} + +# Generate a unique scan ID for today in the format YYYY-MM-DD-NNN. +# The sequence number (NNN) increments based on how many scans already +# ran today (read from history.json). This prevents ID collisions when +# a scanner is triggered multiple times in one day. +# +# Usage: SCAN_ID=$(generate_scan_id "$REPORTS_DIR" "2026-05-21") +# Args: +# $1 - path to the reports directory (must contain history.json) +# $2 - today's date in YYYY-MM-DD format +# Prints: the scan ID (e.g., "2026-05-21-002") +generate_scan_id() { + local reports_dir="$1" + local scan_date="$2" + local last_seq=0 + + # Count how many scans already happened today + if [ -f "$reports_dir/history.json" ]; then + last_seq=$(jq -r \ + --arg date "$scan_date" \ + '[.[] | select(.scan_id | startswith($date))] | length' \ + "$reports_dir/history.json") + fi + + # Zero-padded sequence number + printf "%s-%03d" "$scan_date" $((last_seq + 1)) +} + +# ============================================================================= +# PORTABLE DATE UTILITIES +# ============================================================================= + +# Convert an ISO 8601 timestamp (YYYY-MM-DDTHH:MM:SSZ) to Unix epoch seconds. +# Works on both macOS (BSD date) and Linux (GNU date), with a pure-awk fallback +# that requires no external language runtime. Returns 0 on parse failure. +# +# Usage: epoch=$(iso_to_epoch "2026-06-12T14:00:00Z") +# Args: +# $1 - ISO 8601 timestamp string (must end in Z for UTC) +# Prints: Unix epoch seconds (integer) +iso_to_epoch() { + local timestamp="$1" + + if [ -z "$timestamp" ]; then + echo "0" + return + fi + + # Try GNU date first (Linux) + local epoch + epoch=$(date -d "$timestamp" +%s 2>/dev/null) && { echo "$epoch"; return; } + + # Try BSD date (macOS) — TZ=UTC ensures the Z suffix is honored + epoch=$(TZ=UTC date -j -f "%Y-%m-%dT%H:%M:%SZ" "$timestamp" +%s 2>/dev/null) && { echo "$epoch"; return; } + + # Fallback: pure awk (no external dependencies beyond POSIX) + # Parses YYYY-MM-DDTHH:MM:SSZ and computes epoch via day-counting arithmetic. + epoch=$(echo "$timestamp" | awk -F'[-T:Z]' '{ + if (NF < 6) { print 0; exit } + y = $1 + 0; m = $2 + 0; d = $3 + 0 + H = $4 + 0; M = $5 + 0; S = $6 + 0 + + # Days in each month (non-leap) + split("31,28,31,30,31,30,31,31,30,31,30,31", mdays, ",") + + # Count days from 1970-01-01 + days = 0 + for (yr = 1970; yr < y; yr++) { + days += 365 + if (yr % 4 == 0 && (yr % 100 != 0 || yr % 400 == 0)) days++ + } + for (mo = 1; mo < m; mo++) { + days += mdays[mo] + 0 + if (mo == 2 && y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) days++ + } + days += d - 1 + + print (days * 86400) + (H * 3600) + (M * 60) + S + }' 2>/dev/null) && { echo "$epoch"; return; } + + echo "0" +} + +# Write a file atomically using a temp file and mv. +# Unlike write_report_latest, this takes a source file path (not content string), +# avoiding argument-length limits for large files. +# +# Usage: atomic_write_file "$TMPDIR/state.json" "$REPORTS_DIR/state.json" +# Args: +# $1 - source file path (contents to write) +# $2 - destination file path +atomic_write_file() { + local src="$1" + local dst="$2" + local tmp="${dst}.tmp.$$" + + cp "$src" "$tmp" + mv "$tmp" "$dst" +} + +# ============================================================================= +# JSON SCHEMA VALIDATION +# ============================================================================= + +# Validate that a JSON file matches an expected structure. +# Uses jq to check that required top-level keys exist and are arrays. +# On failure, prints a warning and resets the file to the provided default. +# +# Usage: +# validate_json_schema "$STATE_FILE" \ +# '(.in_progress | type) == "array" and (.reviewed | type) == "array"' \ +# '{"in_progress": [], "reviewed": []}' +# +# Args: +# $1 - path to the JSON file +# $2 - jq boolean expression that must return true for valid files +# $3 - default JSON content to write if validation fails +# Returns: 0 if valid (or reset succeeded), 1 if file cannot be written +validate_json_schema() { + local file="$1" + local check_expr="$2" + local default_content="$3" + + # File does not exist: write default + if [ ! -f "$file" ]; then + printf '%s\n' "$default_content" > "$file" + return 0 + fi + + # File is not valid JSON: reset + if ! jq empty "$file" 2>/dev/null; then + echo "WARN: $file is not valid JSON, resetting to default." >&2 + printf '%s\n' "$default_content" > "$file" + return 0 + fi + + # File does not pass schema check: reset + local valid + valid=$(jq -r "$check_expr" "$file" 2>/dev/null || echo "false") + if [ "$valid" != "true" ]; then + echo "WARN: $file failed schema validation, resetting to default." >&2 + printf '%s\n' "$default_content" > "$file" + return 0 + fi + + return 0 +} + +# ============================================================================= +# DIFF LOGIC +# ============================================================================= + +# Compare current findings against a previous scan to determine what's new, +# what's fixed, and what's recurring. Uses set operations (comm) on sorted keys. +# +# After calling, three files are available in $PROGRAM_TMPDIR: +# new_keys.txt - items in current but not previous (just appeared) +# fixed_keys.txt - items in previous but not current (resolved) +# recurring_keys.txt - items in both (still broken) +# +# Usage: +# read -r new fixed recurring < <(diff_against_previous \ +# "$TMPDIR/current.jsonl" "$TMPDIR/prev.jsonl" '[.repo, .file, .url] | join("|")') +# +# Args: +# $1 - path to current findings file (JSONL, one JSON object per line) +# $2 - path to previous findings file (JSONL) +# $3 - jq expression that extracts the comparison key from each object +# (must produce a single string per line) +# Prints: three space-separated counts: "NEW FIXED RECURRING" +diff_against_previous() { + local current_file="$1" + local prev_file="$2" + local key_expr="$3" + + # Extract and sort keys from both files + jq -r "$key_expr" "$current_file" | sort > "$PROGRAM_TMPDIR/current_keys.txt" + jq -r "$key_expr" "$prev_file" | sort > "$PROGRAM_TMPDIR/prev_keys.txt" + + # Set operations using comm: + # comm -23 = lines only in file 1 (new items) + # comm -13 = lines only in file 2 (fixed items) + # comm -12 = lines in both files (recurring) + comm -23 "$PROGRAM_TMPDIR/current_keys.txt" "$PROGRAM_TMPDIR/prev_keys.txt" \ + > "$PROGRAM_TMPDIR/new_keys.txt" + + comm -13 "$PROGRAM_TMPDIR/current_keys.txt" "$PROGRAM_TMPDIR/prev_keys.txt" \ + > "$PROGRAM_TMPDIR/fixed_keys.txt" + + comm -12 "$PROGRAM_TMPDIR/current_keys.txt" "$PROGRAM_TMPDIR/prev_keys.txt" \ + > "$PROGRAM_TMPDIR/recurring_keys.txt" + + # Count each set + local new_count + local fixed_count + local recurring_count + new_count=$(wc -l < "$PROGRAM_TMPDIR/new_keys.txt" | tr -d ' ') + fixed_count=$(wc -l < "$PROGRAM_TMPDIR/fixed_keys.txt" | tr -d ' ') + recurring_count=$(wc -l < "$PROGRAM_TMPDIR/recurring_keys.txt" | tr -d ' ') + + echo "$new_count $fixed_count $recurring_count" +} + +# ============================================================================= +# REPORTS +# ============================================================================= + +# Overwrite latest.json with the provided JSON content. +# Creates the reports directory if it doesn't exist. +# +# Usage: +# write_report_latest "$REPORTS_DIR" "$json_string" +# +# Args: +# $1 - path to the reports directory +# $2 - JSON content to write (as a string) +# $3 - (optional) filename (default: "latest.json") +write_report_latest() { + local reports_dir="$1" + local content="$2" + local filename="${3:-latest.json}" + + mkdir -p "$reports_dir" + printf '%s\n' "$content" > "$reports_dir/$filename.tmp" && mv "$reports_dir/$filename.tmp" "$reports_dir/$filename" +} + +# Append a row to history.json, keeping the file under a maximum row count. +# If history.json doesn't exist or is empty, initializes it as a JSON array. +# Oldest entries are trimmed when the cap is exceeded. +# +# Usage: +# append_history_row "$REPORTS_DIR" "$history_row_json" 500 +# append_history_row "$REPORTS_DIR" "$row" 500 "fixer-history.json" +# +# Args: +# $1 - path to the reports directory +# $2 - JSON object to append (as a string, e.g., '{"scan_id":"...","date":"..."}') +# $3 - maximum number of rows to keep (default: 500) +# $4 - (optional) filename (default: "history.json") +append_history_row() { + local reports_dir="$1" + local row="$2" + local max_rows="${3:-500}" + local filename="${4:-history.json}" + + mkdir -p "$reports_dir" + + local history_file="$reports_dir/$filename" + + # If history file exists and is non-empty, append and trim + if [ -f "$history_file" ] && [ -s "$history_file" ]; then + local tmp_file="$PROGRAM_TMPDIR/history_new.json" + + jq \ + --argjson row "$row" \ + --argjson cap "$max_rows" \ + '. + [$row] | if length > $cap then .[-$cap:] else . end' \ + "$history_file" > "$tmp_file" + + mv "$tmp_file" "$history_file" + else + # Initialize as a single-element array + echo "[$row]" > "$history_file" + fi +} diff --git a/skills/repoman/scripts/dep-bump-fixer.sh b/skills/repoman/scripts/dep-bump-fixer.sh new file mode 100644 index 0000000..606aabe --- /dev/null +++ b/skills/repoman/scripts/dep-bump-fixer.sh @@ -0,0 +1,648 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Dependency Bump Fixer — kagenti org +# Responds to scanner-created issues with severity-appropriate analysis comments +# on the Dependabot PRs. Does NOT auto-merge — provides actionable commentary. +# +# Usage: +# bash dep-bump-fixer.sh --help +# bash dep-bump-fixer.sh --dry-run +# bash dep-bump-fixer.sh --live --issue-limit 3 +# ============================================================================= + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- CLI args --- +DRY_RUN=true # Safe by default +ISSUE_LIMIT=15 +VERBOSE=false +SHOW_HELP=false + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) DRY_RUN=true; shift ;; + --live) DRY_RUN=false; shift ;; + --issue-limit) ISSUE_LIMIT="$2"; shift 2 ;; + --verbose) VERBOSE=true; shift ;; + --help|-h) SHOW_HELP=true; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if [ "$SHOW_HELP" = true ]; then + cat << 'USAGE' +dep-bump-fixer -- Analyze stale Dependabot PRs and post actionable comments + +USAGE: + dep-bump-fixer.sh [OPTIONS] + +OPTIONS: + --dry-run Analyze and preview comments only (default) + --live Post comments on PRs and issues + --issue-limit N Process at most N issues (default: 5) + --verbose Print additional diagnostic output + --help, -h Show this help + +ENVIRONMENT: + REPOS_DIR (required) Directory containing cloned org repos + REPORTS_DIR (optional) Where to write reports (default: ~/reports/dep-bump) + +PREREQUISITES: + bash 3.2+, gh (authenticated), jq +USAGE + exit 0 +fi + +# Resolve deployment constants (repos_dir, fork_owner) from ~/.repoman/config.json. +repoman_config + +# --- Configuration --- +validate_repos_dir "${REPOS_DIR:-}" + +REPORTS_DIR="${REPORTS_DIR:-$HOME/reports/dep-bump}" +SCAN_DATE=$(date -u +"%Y-%m-%d") +SCAN_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +MAX_HISTORY_ROWS=500 +# Provenance footer appended to every fixer comment/issue. Resolved once at +# startup: the helper lazily writes _meta..json beside this skill and returns the +# pinned-SHA RepoMan attribution (blob/main fallback when unresolved). +SKILL_FOOTER=$(skill_attribution "dep-bump-fixer") +# Stable dedup key: the substring every footer form shares, regardless of the +# per-run SHA (both "...RepoMan dep-bump-fixer@..." and the blob/main +# fallback contain it). The dedup search (below) looks for THIS, not the whole +# footer, so a changing SHA never defeats dedup. Comments carrying the former +# marker value are no longer recognized, so an already-analyzed PR may receive +# one duplicate comment on the next run after the marker changed — acceptable, +# one-time. +FIXER_SIGNATURE="RepoMan dep-bump-fixer" + +# --- Workspace setup --- +setup_workspace "dep-bump-fixer" +TMPDIR="$PROGRAM_TMPDIR" +mkdir -p "$REPORTS_DIR" + +# --- Scan ID --- +SCAN_ID=$(generate_scan_id "$REPORTS_DIR" "$SCAN_DATE") + +echo "=== Dep Bump Fixer $SCAN_ID ===" +echo "Repos dir: $REPOS_DIR" +echo "Reports dir: $REPORTS_DIR" +if [ "$DRY_RUN" = true ]; then echo "Mode: DRY RUN (no comments posted)"; else echo "Mode: LIVE"; fi +echo "Issue limit: $ISSUE_LIMIT" + +# --- Step 2: Capture baseline (first run only) --- +if [ ! -f "$REPORTS_DIR/baseline.json" ]; then + echo "" + echo "--- Capturing baseline (first run) ---" + + # Query merged Dependabot PRs across all repos (last 90 days) + : > "$TMPDIR/merged_prs.jsonl" + # Drive the loop from the enrolled set (authoritative), statting each clone, + # rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently + # drops a ".github" repo. The here-string keeps the body in the current shell. + ENROLLED=$(repoman_load_enrolled) || exit 1 + CLONES=$(enrolled_clone_dirs "$ENROLLED") + while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + + gh pr list --repo "$full_repo" \ + --author "app/dependabot" \ + --state merged \ + --json number,createdAt,mergedAt \ + --limit 50 2>/dev/null | jq -c --arg repo "$full_repo" \ + '.[] | . + {repo: $repo}' >> "$TMPDIR/merged_prs.jsonl" 2>/dev/null || true + + sleep 0.5 + done <<< "$CLONES" + + # Compute baseline metrics + total_merged=$(wc -l < "$TMPDIR/merged_prs.jsonl" | tr -d ' ') + + # Median time-to-merge (days) using jq epoch arithmetic + median_ttm=$(jq -s ' + [.[] | + ((.mergedAt | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) - + (.createdAt | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 86400 | floor + ] | sort | if length == 0 then 0 + elif length % 2 == 1 then .[length / 2 | floor] + else (.[length / 2 - 1] + .[length / 2]) / 2 + end + ' "$TMPDIR/merged_prs.jsonl" 2>/dev/null || echo "0") + + # Current stale count from scanner + stale_count=0 + if [ -f "$REPORTS_DIR/latest.json" ]; then + stale_count=$(jq '.stale_prs | length' "$REPORTS_DIR/latest.json" 2>/dev/null || echo "0") + fi + + jq -nc \ + --arg date "$SCAN_TIME" \ + --argjson total_merged "$total_merged" \ + --argjson median_ttm "$median_ttm" \ + --argjson stale_count "$stale_count" \ + '{captured_at: $date, total_merged_90d: $total_merged, + median_time_to_merge_days: $median_ttm, stale_count_at_baseline: $stale_count}' \ + > "$REPORTS_DIR/baseline.json" + + echo "Baseline captured: $total_merged merged PRs (90d), median TTM ${median_ttm}d, $stale_count stale" +fi + +# --- Step 3: Discover scanner-created issues --- +echo "" +echo "--- Discovering scanner issues ---" + +: > "$TMPDIR/issues.jsonl" +REPOS_CHECKED=0 + +# Drive the loop from the enrolled set (authoritative), statting each clone, +# rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently +# drops a ".github" repo. enrolled_clone_dirs emits each enrolled owner/name +# whose clone exists; the here-string keeps the body in the current shell so +# counters accumulate. +ENROLLED=$(repoman_load_enrolled) || exit 1 +CLONES=$(enrolled_clone_dirs "$ENROLLED") + +while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + + REPOS_CHECKED=$((REPOS_CHECKED + 1)) + + issues_json=$(gh issue list --repo "$full_repo" \ + --search "[dep-bump] in:title" \ + --state open --limit 100 \ + --json number,title,body 2>/dev/null || echo "[]") + + issue_count=$(echo "$issues_json" | jq 'length') + if [ "$issue_count" -gt 0 ]; then + echo "$issues_json" | jq -c --arg repo "$full_repo" '.[] | . + {repo: $repo}' \ + >> "$TMPDIR/issues.jsonl" + fi + + sleep 0.3 +done <<< "$CLONES" + +TOTAL_ISSUES=$(wc -l < "$TMPDIR/issues.jsonl" | tr -d ' ') +echo "Found $TOTAL_ISSUES open scanner issues across $REPOS_CHECKED repos" + +# --- Step 4-7: Process each issue --- +echo "" +echo "--- Processing issues ---" + +ISSUES_PROCESSED=0 +COMMENTS_POSTED=0 +ISSUES_CLOSED=0 +SECURITY_COUNT=0 +ROUTINE_COUNT=0 +MAJOR_COUNT=0 +SKIPPED_ALREADY_COMMENTED=0 +SKIPPED_PR_CLOSED=0 + +while IFS= read -r issue_record; do + # Check issue limit + if [ "$ISSUES_PROCESSED" -ge "$ISSUE_LIMIT" ]; then + echo " Issue limit ($ISSUE_LIMIT) reached, stopping" + break + fi + + issue_number=$(echo "$issue_record" | jq -r '.number') + issue_repo=$(echo "$issue_record" | jq -r '.repo') + body=$(echo "$issue_record" | jq -r '.body') + + # --- Parse issue body --- + pr_number=$(echo "$body" | sed -nE 's/^\*\*PR:\*\*[[:space:]]*#([0-9]+)/\1/p' | head -1) + package=$(echo "$body" | sed -nE 's/^\*\*Package:\*\*[[:space:]]*(.*)/\1/p' | head -1 | sed 's/[[:space:]]*$//') + severity=$(echo "$body" | sed -nE 's/^\*\*Severity:\*\*[[:space:]]*(.*)/\1/p' | head -1 | sed 's/[[:space:]]*$//') + sla_days=$(echo "$body" | sed -nE 's/^\*\*SLA:\*\*[[:space:]]*([0-9]+).*/\1/p' | head -1) + age_days=$(echo "$body" | sed -nE 's/^\*\*Age:\*\*[[:space:]]*([0-9]+).*/\1/p' | head -1) + ecosystem=$(echo "$body" | sed -nE 's/^\*\*Ecosystem:\*\*[[:space:]]*(.*)/\1/p' | head -1 | sed 's/[[:space:]]*$//') + category=$(echo "$body" | sed -nE 's/^Category:[[:space:]]*(.*)/\1/p' | head -1 | sed 's/[[:space:]]*$//') + version_str=$(echo "$body" | sed -nE 's/^\*\*Version:\*\*[[:space:]]*(.*)/\1/p' | head -1 | sed 's/[[:space:]]*$//') + + # Validate required fields + if [ -z "$pr_number" ] || [ -z "$package" ] || [ -z "$severity" ]; then + if [ "$VERBOSE" = true ]; then + echo " SKIP #$issue_number: missing required fields (pr=$pr_number, pkg=$package, sev=$severity)" + fi + continue + fi + + # Validate pr_number is numeric + if ! echo "$pr_number" | grep -qE '^[0-9]+$'; then + if [ "$VERBOSE" = true ]; then + echo " SKIP #$issue_number: pr_number not numeric ($pr_number)" + fi + continue + fi + + echo " Processing #$issue_number: $package in $issue_repo (PR #$pr_number, $severity)" + + # --- Step 5: Check PR state --- + pr_state=$(gh_with_backoff pr view "$pr_number" --repo "$issue_repo" --json state --jq '.state' 2>/dev/null || echo "UNKNOWN") + + if [ "$pr_state" = "MERGED" ] || [ "$pr_state" = "CLOSED" ]; then + echo " PR #$pr_number is $pr_state — closing issue" + SKIPPED_PR_CLOSED=$((SKIPPED_PR_CLOSED + 1)) + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would close issue #$issue_number" + else + close_issue_if_valid "$issue_repo" "$issue_number" \ + "Dependabot PR #$pr_number has been ${pr_state}. Auto-closing." || true + fi + ISSUES_CLOSED=$((ISSUES_CLOSED + 1)) + continue + fi + + # Check if fixer already commented on this PR + # $sig below is a jq variable passed via --arg, not a shell expansion + # shellcheck disable=SC2016 + already_commented=$(gh_with_backoff pr view "$pr_number" --repo "$issue_repo" \ + --json comments --jq --arg sig "$FIXER_SIGNATURE" \ + '[.comments[] | select(.body | contains($sig))] | length' 2>/dev/null || echo "0") + + if [ "$already_commented" -gt 0 ]; then + if [ "$VERBOSE" = true ]; then + echo " Already commented, skipping" + fi + SKIPPED_ALREADY_COMMENTED=$((SKIPPED_ALREADY_COMMENTED + 1)) + continue + fi + + ISSUES_PROCESSED=$((ISSUES_PROCESSED + 1)) + + # Re-check CI status from live PR + live_ci="unknown" + ci_checks=$(gh_with_backoff pr checks "$pr_number" --repo "$issue_repo" 2>/dev/null || echo "") + if [ -n "$ci_checks" ]; then + if echo "$ci_checks" | grep -qiF "fail"; then + live_ci="failing" + elif echo "$ci_checks" | grep -qiF "pass"; then + live_ci="passing" + elif echo "$ci_checks" | grep -qiF "pending"; then + live_ci="pending" + fi + fi + + # Get PR body for changelog extraction + pr_body=$(gh_with_backoff pr view "$pr_number" --repo "$issue_repo" --json body --jq '.body' 2>/dev/null || echo "") + + # Extract changelog/release notes section from PR body + changelog_summary="" + if [ -n "$pr_body" ]; then + # Dependabot PR bodies have sections like "Release notes" or "Changelog" or "Commits" + changelog_summary=$(echo "$pr_body" | sed -n '/Release notes/,/^<\/details>/p' | head -30 || true) + if [ -z "$changelog_summary" ]; then + changelog_summary=$(echo "$pr_body" | sed -n '/Changelog/,/^<\/details>/p' | head -30 || true) + fi + if [ -z "$changelog_summary" ]; then + changelog_summary="(No changelog section found in PR body)" + fi + # Truncate if too long + if [ ${#changelog_summary} -gt 1500 ]; then + changelog_summary="${changelog_summary:0:1500}..." + fi + fi + + # Extract CVE/GHSA references + cve_refs="" + if [ -n "$pr_body" ]; then + cve_refs=$(echo "$pr_body" | grep -oE '(CVE-[0-9]{4}-[0-9]+|GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4})' | sort -u | head -5 | tr '\n' ', ' | sed 's/,$//' || true) + fi + + # Compute overdue days + overdue=0 + if [ -n "$age_days" ] && [ -n "$sla_days" ]; then + overdue=$((age_days - sla_days)) + fi + + # --- Step 6: Generate analysis comment --- + comment="" + + case "$category" in + security) + SECURITY_COUNT=$((SECURITY_COUNT + 1)) + comment="## Security Bump Escalation + +**Package:** $package ($version_str) +**Severity:** $severity | **SLA:** ${sla_days} days | **Overdue:** ${overdue} days +**CI Status:** $live_ci + +### Advisory Context +" + if [ -n "$cve_refs" ]; then + comment="${comment}Vulnerability references: $cve_refs +" + fi + comment="${comment} +$changelog_summary + +### Action Required +This PR has exceeded the ${sla_days}-day SLA for ${severity}-severity patches. Please review and merge, or document a deferral reason. + +--- +_${SKILL_FOOTER} (scan $SCAN_ID)_" + ;; + + major) + MAJOR_COUNT=$((MAJOR_COUNT + 1)) + comment="## Major Version Bump Analysis + +**Package:** $package ($version_str) +**Ecosystem:** $ecosystem | **Age:** ${age_days} days + +### Migration Notes +$changelog_summary + +### Recommendation +Major version bumps require manual review. Consider: +- Bundling with related dependency updates +- Scheduling migration work if breaking changes affect multiple files +- Deferring with documented justification if migration is non-trivial + +--- +_${SKILL_FOOTER} (scan $SCAN_ID)_" + ;; + + *) + ROUTINE_COUNT=$((ROUTINE_COUNT + 1)) + # Detect breaking change indicators in changelog + breaking_changes="None detected" + if [ -n "$changelog_summary" ]; then + if echo "$changelog_summary" | grep -qiE 'breaking|BREAKING|deprecat'; then + breaking_changes="Possible breaking changes detected in changelog — review recommended" + fi + fi + + recommendation="Safe to merge (minor/patch, CI $live_ci, no breaking changes detected)" + if [ "$live_ci" = "failing" ]; then + recommendation="Review needed — CI is failing. Check if failure is pre-existing on main or caused by this bump." + elif [ "$breaking_changes" != "None detected" ]; then + recommendation="Review recommended — possible breaking changes in changelog" + fi + + comment="## Dependency Update Analysis + +**Package:** $package ($version_str) +**Ecosystem:** $ecosystem | **Age:** ${age_days} days (SLA: ${sla_days} days) +**CI Status:** $live_ci + +### Changelog Summary +$changelog_summary + +### Risk Assessment +- Breaking changes: $breaking_changes +- Recommendation: $recommendation + +--- +_${SKILL_FOOTER} (scan $SCAN_ID)_" + ;; + esac + + # --- Step 7: Post comment --- + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would post comment on PR #$pr_number ($category)" + if [ "$VERBOSE" = true ]; then + echo " --- Comment preview ---" + echo "$comment" | head -20 + echo " --- (truncated) ---" + fi + else + if gh_with_backoff pr comment "$pr_number" --repo "$issue_repo" --body "$comment" 2>/dev/null; then + COMMENTS_POSTED=$((COMMENTS_POSTED + 1)) + echo " Posted analysis on PR #$pr_number" + + # Also comment on the scanner issue + gh_with_backoff issue comment "$issue_number" --repo "$issue_repo" \ + --body "Analysis posted on PR #$pr_number. Awaiting human action. + +_${SKILL_FOOTER} (scan $SCAN_ID)_" 2>/dev/null || true + else + echo " WARN: Failed to post comment on PR #$pr_number" + fi + fi + + sleep 1 +done < "$TMPDIR/issues.jsonl" + +# --- Step 8: Handle repos missing Dependabot config --- +CONFIG_PRS_CREATED=0 + +if [ -f "$REPORTS_DIR/latest.json" ]; then + coverage_gaps=$(jq -c '.coverage_gaps[]? | select(.has_config == false)' "$REPORTS_DIR/latest.json" 2>/dev/null || true) + + if [ -n "$coverage_gaps" ]; then + echo "" + echo "--- Repos missing Dependabot config ---" + + while IFS= read -r gap; do + [ -z "$gap" ] && continue + + # Respect issue limit for config PRs too + if [ "$ISSUES_PROCESSED" -ge "$ISSUE_LIMIT" ]; then + break + fi + + gap_repo=$(echo "$gap" | jq -r '.repo') + gap_ecosystems=$(echo "$gap" | jq -r '.ecosystems | join(",")') + + echo " $gap_repo: missing config (ecosystems: $gap_ecosystems)" + + # Generate dependabot.yml content + dependabot_yml="version: 2\nupdates:" + IFS=',' read -ra eco_arr <<< "$gap_ecosystems" + for eco in "${eco_arr[@]}"; do + [ -z "$eco" ] && continue + dependabot_yml="${dependabot_yml}\n - package-ecosystem: \"$eco\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n groups:\n minor-and-patch:\n update-types:\n - \"minor\"\n - \"patch\"" + done + + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would create dependabot.yml PR for $gap_repo" + if [ "$VERBOSE" = true ]; then + printf " Config:\n%b\n" "$dependabot_yml" | head -20 + fi + else + # Create config via fork-based PR + repo_dir="$REPOS_DIR/$gap_repo" + if [ -d "$repo_dir/.git" ]; then + branch_name="chore/add-dependabot-config" + + # Check if PR already exists + existing_pr=$(gh pr list --repo "$gap_repo" --state open \ + --search "dependabot.yml in:title" --json number --jq '.[0].number' 2>/dev/null || echo "") + + if [ -n "$existing_pr" ] && [ "$existing_pr" != "null" ]; then + echo " PR #$existing_pr already exists for dependabot config" + continue + fi + + # Write config file + config_dir="$repo_dir/.github" + mkdir -p "$config_dir" + printf '%b\n' "$dependabot_yml" > "$config_dir/dependabot.yml" + + # Use fork-based PR workflow + pr_created=$( + cd "$repo_dir" || exit 1 + git checkout -B "$branch_name" origin/main 2>/dev/null || git checkout -B "$branch_name" main 2>/dev/null || true + git add .github/dependabot.yml + if git diff --cached --quiet; then + echo "no" + else + ensure_fork "${gap_repo%%/*}" "${gap_repo#*/}" "$FORK_OWNER" + if create_fork_pr "${gap_repo%%/*}" "${gap_repo#*/}" "$FORK_OWNER" "$branch_name" \ + "chore: Add Dependabot configuration" \ + "chore: Add Dependabot configuration for $gap_repo" \ + "Enable automated dependency updates with weekly schedule and grouped minor/patch updates. + +Ecosystems: $gap_ecosystems + +--- +_${SKILL_FOOTER} (scan $SCAN_ID)_"; then + echo "yes" + else + echo "no" + fi + fi + ) 2>/dev/null || echo "no" + if [ "$pr_created" = "yes" ]; then + CONFIG_PRS_CREATED=$((CONFIG_PRS_CREATED + 1)) + fi + fi + fi + + ISSUES_PROCESSED=$((ISSUES_PROCESSED + 1)) + sleep 1 + done <<< "$coverage_gaps" + fi +fi + +# --- Step 9: Compute metrics --- +echo "" +echo "--- Computing metrics ---" + +# Query recently merged Dependabot PRs (last 30 days) for TTM +: > "$TMPDIR/recent_merged.jsonl" +# Drive the loop from the enrolled set (authoritative), statting each clone, +# rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently +# drops a ".github" repo. The here-string keeps the body in the current shell. +ENROLLED=$(repoman_load_enrolled) || exit 1 +CLONES=$(enrolled_clone_dirs "$ENROLLED") +while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + + gh pr list --repo "$full_repo" \ + --author "app/dependabot" \ + --state merged \ + --json number,createdAt,mergedAt \ + --limit 20 2>/dev/null | jq -c --arg repo "$full_repo" \ + '.[] | . + {repo: $repo}' >> "$TMPDIR/recent_merged.jsonl" 2>/dev/null || true + + sleep 0.3 +done <<< "$CLONES" + +# Compute median time-to-merge +MERGED_SINCE_LAST=0 +MEDIAN_TTM=0 + +if [ -s "$TMPDIR/recent_merged.jsonl" ]; then + MERGED_SINCE_LAST=$(wc -l < "$TMPDIR/recent_merged.jsonl" | tr -d ' ') + MEDIAN_TTM=$(jq -s ' + [.[] | + ((.mergedAt | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) - + (.createdAt | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 86400 | floor + ] | sort | if length == 0 then 0 + elif length % 2 == 1 then .[length / 2 | floor] + else (.[length / 2 - 1] + .[length / 2]) / 2 + end + ' "$TMPDIR/recent_merged.jsonl" 2>/dev/null || echo "0") +fi + +# Current stale count +CURRENT_STALE=0 +if [ -f "$REPORTS_DIR/latest.json" ]; then + CURRENT_STALE=$(jq '.stale_prs | length' "$REPORTS_DIR/latest.json" 2>/dev/null || echo "0") +fi + +# Previous fixer run metrics for delta +PREV_STALE=0 +if [ -f "$REPORTS_DIR/fixer-latest.json" ]; then + PREV_STALE=$(jq '.metrics.stale_count // 0' "$REPORTS_DIR/fixer-latest.json" 2>/dev/null || echo "0") +fi + +# Baseline metrics +BASELINE_TTM=0 +BASELINE_STALE=0 +if [ -f "$REPORTS_DIR/baseline.json" ]; then + BASELINE_TTM=$(jq '.median_time_to_merge_days // 0' "$REPORTS_DIR/baseline.json" 2>/dev/null || echo "0") + BASELINE_STALE=$(jq '.stale_count_at_baseline // 0' "$REPORTS_DIR/baseline.json" 2>/dev/null || echo "0") +fi + +STALE_DELTA=$((CURRENT_STALE - PREV_STALE)) + +# --- Write reports --- +FIXER_LATEST=$(jq -nc \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson duration "$SECONDS" \ + --argjson issues_processed "$ISSUES_PROCESSED" \ + --argjson comments_posted "$COMMENTS_POSTED" \ + --argjson issues_closed "$ISSUES_CLOSED" \ + --argjson config_prs "$CONFIG_PRS_CREATED" \ + --argjson security "$SECURITY_COUNT" \ + --argjson routine "$ROUTINE_COUNT" \ + --argjson major "$MAJOR_COUNT" \ + --argjson skipped_commented "$SKIPPED_ALREADY_COMMENTED" \ + --argjson skipped_closed "$SKIPPED_PR_CLOSED" \ + --argjson merged_recent "$MERGED_SINCE_LAST" \ + --argjson median_ttm "$MEDIAN_TTM" \ + --argjson stale_count "$CURRENT_STALE" \ + --argjson baseline_ttm "$BASELINE_TTM" \ + --argjson baseline_stale "$BASELINE_STALE" \ + '{scan_id: $scan_id, date: $date, duration_seconds: $duration, + issues_processed: $issues_processed, comments_posted: $comments_posted, + issues_closed: $issues_closed, config_prs_created: $config_prs, + breakdown: {security: $security, routine: $routine, major: $major}, + skipped: {already_commented: $skipped_commented, pr_closed: $skipped_closed}, + metrics: {merged_recent: $merged_recent, median_time_to_merge: $median_ttm, + stale_count: $stale_count, baseline_ttm: $baseline_ttm, baseline_stale: $baseline_stale}}') + +write_report_latest "$REPORTS_DIR" "$FIXER_LATEST" "fixer-latest.json" + +HISTORY_ROW=$(jq -nc \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson issues_processed "$ISSUES_PROCESSED" \ + --argjson comments_posted "$COMMENTS_POSTED" \ + --argjson issues_closed "$ISSUES_CLOSED" \ + --argjson median_ttm "$MEDIAN_TTM" \ + --argjson stale_count "$CURRENT_STALE" \ + '{scan_id: $scan_id, date: $date, issues_processed: $issues_processed, + comments_posted: $comments_posted, issues_closed: $issues_closed, + median_ttm: $median_ttm, stale_count: $stale_count}') + +append_history_row "$REPORTS_DIR" "$HISTORY_ROW" "$MAX_HISTORY_ROWS" "fixer-history.json" + +echo "Wrote $REPORTS_DIR/fixer-latest.json" +echo "Appended to $REPORTS_DIR/fixer-history.json" + +# --- Summary --- +echo "" +echo "=== Dep Bump Fixer $SCAN_ID Summary ===" +echo "Issues processed: $ISSUES_PROCESSED" +echo " Security escalations: $SECURITY_COUNT" +echo " Routine analyses: $ROUTINE_COUNT" +echo " Major bump analyses: $MAJOR_COUNT" +if [ "$DRY_RUN" = true ]; then + echo "Comments posted: 0 (DRY RUN)" +else + echo "Comments posted: $COMMENTS_POSTED" +fi +echo "Issues closed (PR merged/closed): $ISSUES_CLOSED" +echo "Config PRs created: $CONFIG_PRS_CREATED" +echo "Skipped: $SKIPPED_ALREADY_COMMENTED already commented, $SKIPPED_PR_CLOSED PR closed" +echo "Metrics: median TTM ${MEDIAN_TTM}d (baseline: ${BASELINE_TTM}d), stale $CURRENT_STALE (delta: ${STALE_DELTA})" +echo "Duration: ${SECONDS}s" diff --git a/skills/repoman/scripts/dep-bump-scanner.sh b/skills/repoman/scripts/dep-bump-scanner.sh new file mode 100644 index 0000000..ab12542 --- /dev/null +++ b/skills/repoman/scripts/dep-bump-scanner.sh @@ -0,0 +1,705 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Dependency Bump Scanner +# Monitors Dependabot PRs, classifies by severity, flags SLA breaches, +# creates/closes GitHub issues, writes reports. +# +# Usage: +# bash dep-bump-scanner.sh --help +# bash dep-bump-scanner.sh --dry-run +# bash dep-bump-scanner.sh --issue-limit 3 +# ============================================================================= + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- CLI args --- +DRY_RUN=false +ISSUE_LIMIT=0 # 0 = unlimited +SHOW_HELP=false + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) DRY_RUN=true; shift ;; + --issue-limit) ISSUE_LIMIT="$2"; shift 2 ;; + --help|-h) SHOW_HELP=true; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if [ "$SHOW_HELP" = true ]; then + cat << 'USAGE' +dep-bump-scanner -- Monitor Dependabot PRs for SLA breaches + +USAGE: + dep-bump-scanner.sh [OPTIONS] + +OPTIONS: + --dry-run Scan and report only; do not create/close issues + --issue-limit N Create at most N issues per run (0 = unlimited) + --help, -h Show this help + +ENVIRONMENT: + REPOS_DIR (required) Directory containing cloned org repos + REPORTS_DIR (optional) Where to write reports (default: ~/reports/dep-bump) + +PREREQUISITES: + bash 3.2+, gh (authenticated), jq + +SEVERITY SLAs: + Critical (CVSS 9.0+) 3 days + High (CVSS 7.0-8.9) 7 days + Medium (CVSS 4.0-6.9) 30 days + Major version bump 30 days + Routine (no CVE) 14 days +USAGE + exit 0 +fi + +# Resolve deployment constants (repos_dir, fork_owner) from ~/.repoman/config.json. +repoman_config + +# --- Configuration --- +validate_repos_dir "${REPOS_DIR:-}" + +REPORTS_DIR="${REPORTS_DIR:-$HOME/reports/dep-bump}" +SCAN_DATE=$(date -u +"%Y-%m-%d") +SCAN_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +MAX_HISTORY_ROWS=500 +ESCALATION_THRESHOLD=5 + +# Provenance footer for the issue bodies this scanner creates. Resolved once at +# startup: the helper lazily writes _meta..json beside this skill and returns the +# pinned-SHA RepoMan attribution (blob/main fallback when the SHA is unresolved). +SKILL_FOOTER=$(skill_attribution "dep-bump-scanner") + +# SLA thresholds (days) +SLA_CRITICAL=3 +SLA_HIGH=7 +SLA_MEDIUM=30 +SLA_ROUTINE=14 +SLA_MAJOR=30 + +# --- Workspace setup --- +setup_workspace "dep-bump" +TMPDIR="$PROGRAM_TMPDIR" +mkdir -p "$REPORTS_DIR" + +# --- Scan ID --- +SCAN_ID=$(generate_scan_id "$REPORTS_DIR" "$SCAN_DATE") + +echo "=== Dep Bump Scan $SCAN_ID ===" +echo "Repos dir: $REPOS_DIR" +echo "Reports dir: $REPORTS_DIR" +if [ "$DRY_RUN" = true ]; then echo "Mode: DRY RUN (no issues)"; fi +if [ "$ISSUE_LIMIT" -gt 0 ]; then echo "Issue limit: $ISSUE_LIMIT"; fi + +# --- Step 1: Ecosystem detection --- +echo "" +echo "--- Detecting ecosystems ---" + +: > "$TMPDIR/ecosystems.jsonl" +REPOS_SCANNED=0 + +# Drive the loop from the enrolled set (authoritative), statting each clone, +# rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently +# drops a ".github" repo. enrolled_clone_dirs emits each enrolled owner/name +# whose clone exists; the here-string keeps the body in the current shell so +# counters accumulate. +ENROLLED=$(repoman_load_enrolled) || exit 1 +CLONES=$(enrolled_clone_dirs "$ENROLLED") + +while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + repo_dir="$REPOS_DIR/$full_repo" + + REPOS_SCANNED=$((REPOS_SCANNED + 1)) + ecosystems="" + + # Detect Python + if [ -f "$repo_dir/pyproject.toml" ] || [ -f "$repo_dir/setup.py" ] || [ -f "$repo_dir/requirements.txt" ]; then + ecosystems="${ecosystems}pip," + fi + + # Detect Node + if [ -f "$repo_dir/package.json" ]; then + ecosystems="${ecosystems}npm," + fi + + # Detect Go + if [ -f "$repo_dir/go.mod" ]; then + ecosystems="${ecosystems}gomod," + fi + + # Detect Rust + if [ -f "$repo_dir/Cargo.toml" ]; then + ecosystems="${ecosystems}cargo," + fi + + # Detect Docker (check recursively for Dockerfiles) + if find "$repo_dir" -maxdepth 3 -name "Dockerfile" -print -quit 2>/dev/null | grep -q .; then + ecosystems="${ecosystems}docker," + fi + + # Detect GitHub Actions + if find "$repo_dir/.github/workflows" -maxdepth 1 -name "*.yml" -print -quit 2>/dev/null | grep -q .; then + ecosystems="${ecosystems}github-actions," + fi + + # Strip trailing comma + ecosystems="${ecosystems%,}" + + # Store the full owner/name so Step 2 builds correct refs directly. + jq -nc --arg repo "$full_repo" --arg eco "$ecosystems" \ + '{repo: $repo, ecosystems: ($eco | split(",") | map(select(. != "")))}' \ + >> "$TMPDIR/ecosystems.jsonl" +done <<< "$CLONES" + +echo "Detected ecosystems for $REPOS_SCANNED repos" + +# --- Step 2: List Dependabot PRs and classify --- +echo "" +echo "--- Listing Dependabot PRs ---" + +: > "$TMPDIR/all_prs.jsonl" +# Per-repo open-PR breakdown ({repo, open_prs}) so a subset run can carry +# unscanned repos' share of the population totals forward and recompute a +# consistent whole-set total_open_prs/repos_with_dependabot (see the subset +# totals block below). One line per repo with >=1 open Dependabot PR. +: > "$TMPDIR/repo_pr_counts.jsonl" +REPOS_WITH_DEPENDABOT=0 +TOTAL_OPEN_PRS=0 + +while IFS= read -r eco_record; do + full_repo=$(echo "$eco_record" | jq -r '.repo') # full owner/name from Step 1 + + echo " Checking $full_repo..." + + # List open Dependabot PRs. Query the canonical repo directly -- gh pr list + # with --author silently returns empty across a rename redirect, which is why + # the pre-rename names reported zero Dependabot PRs. + prs_json=$(gh pr list --repo "$full_repo" \ + --author "app/dependabot" \ + --state open \ + --json number,title,createdAt,labels,body,statusCheckRollup \ + --limit 100 2>/dev/null || echo "[]") + + pr_count=$(echo "$prs_json" | jq 'length') + if [ "$pr_count" -eq 0 ]; then + continue + fi + + REPOS_WITH_DEPENDABOT=$((REPOS_WITH_DEPENDABOT + 1)) + TOTAL_OPEN_PRS=$((TOTAL_OPEN_PRS + pr_count)) + jq -nc --arg repo "$full_repo" --argjson n "$pr_count" '{repo: $repo, open_prs: $n}' \ + >> "$TMPDIR/repo_pr_counts.jsonl" + + # Try to get dependabot alerts for severity lookup (graceful failure) + alerts_json="[]" + if alerts_raw=$(gh api "/repos/$full_repo/dependabot/alerts" 2>/dev/null); then + alerts_json=$(echo "$alerts_raw" | jq -c \ + '[.[] | select(.state == "open") | {package: .dependency.package.name, severity: .security_advisory.severity}]' \ + 2>/dev/null || echo "[]") + fi + + # Process each PR: classify severity, compute age, determine staleness + echo "$prs_json" | jq -c --arg repo "$full_repo" \ + --arg scan_date "$SCAN_DATE" \ + --argjson sla_critical "$SLA_CRITICAL" \ + --argjson sla_high "$SLA_HIGH" \ + --argjson sla_medium "$SLA_MEDIUM" \ + --argjson sla_routine "$SLA_ROUTINE" \ + --argjson sla_major "$SLA_MAJOR" \ + --argjson alerts "$alerts_json" ' + .[] | + + # Parse title for package and versions + # Handles: "Bump X from A to B", "chore(deps): bump X from A to B in /dir", grouped updates + (.title | capture("[Bb]ump (?[^ ]+) from (?[^ ]+) to (?[^ ]+)") // null) as $single | + (.title | capture("[Bb]ump the (?[^ ]+) group") // null) as $group | + (if $single != null then $single.pkg + elif $group != null then ($group.grp + " group") + else (.title | sub("^[^:]+:\\s*"; "")) + end) as $package | + (if $single != null then $single.from else "" end) as $from_ver | + (if $single != null then $single.to else "" end) as $to_ver | + + # Detect major version bump + (if ($from_ver != "" and $to_ver != "") then + (($from_ver | split(".") | .[0] | tonumber) // -1) as $major_from | + (($to_ver | split(".") | .[0] | tonumber) // -1) as $major_to | + ($major_to > $major_from and $major_from >= 0) + else false end) as $is_major | + + # Security signals + ((.labels // []) | map(.name) | any(. == "security" or . == "Security")) as $has_security_label | + ((.body // "") | test("CVE-|GHSA-")) as $has_cve | + + # Check dependabot alerts for this package + ($alerts | map(select(.package == $package)) | .[0].severity // null) as $alert_severity | + + # Classify severity (priority order) + (if $alert_severity == "critical" then "critical" + elif $alert_severity == "high" then "high" + elif ($has_security_label or $has_cve) then "high" + elif $alert_severity == "medium" then "medium" + elif $is_major then "major" + else "routine" + end) as $severity | + + # SLA lookup + (if $severity == "critical" then $sla_critical + elif $severity == "high" then $sla_high + elif $severity == "medium" then $sla_medium + elif $severity == "major" then $sla_major + else $sla_routine end) as $sla | + + # Compute age in days + ((.createdAt | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) as $created_epoch | + ($scan_date + "T00:00:00Z" | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) as $today_epoch | + (($today_epoch - $created_epoch) / 86400 | floor) + ) as $age | + + # CI status + ((.statusCheckRollup // []) | + if length == 0 then "pending" + elif all(.conclusion == "SUCCESS") then "passing" + else "failing" + end) as $ci_status | + + # Category for issue body + (if ($has_security_label or $has_cve or $alert_severity != null) then "security" + elif $is_major then "major" + else "routine" + end) as $category | + + # Ecosystem detection from title/files heuristic + (if (.title | test("(?i)docker|container|image")) then "docker" + elif (.title | test("(?i)actions/|github-actions")) then "github-actions" + elif (.title | test("(?i)\\.js|npm|node|typescript")) then "npm" + elif (.title | test("(?i)go\\.|golang")) then "gomod" + elif (.title | test("(?i)cargo|crate")) then "cargo" + else "pip" + end) as $ecosystem | + + { + repo: $repo, + pr_number: .number, + title: .title, + package: $package, + from_version: $from_ver, + to_version: $to_ver, + ecosystem: $ecosystem, + severity: $severity, + sla_days: $sla, + age_days: $age, + is_stale: ($age > $sla), + has_cve: $has_cve, + ci_status: $ci_status, + category: $category, + created_at: .createdAt + } + ' >> "$TMPDIR/all_prs.jsonl" 2>/dev/null || true + + sleep 0.5 +done < "$TMPDIR/ecosystems.jsonl" + +echo "Repos with Dependabot PRs: $REPOS_WITH_DEPENDABOT" +echo "Total open PRs: $TOTAL_OPEN_PRS" + +# --- Step 3: Coverage audit --- +echo "" +echo "--- Dependabot coverage audit ---" + +: > "$TMPDIR/coverage_gaps.jsonl" + +while IFS= read -r eco_record; do + # .repo is the full owner/name; the clone lives at $REPOS_DIR//. + full_repo=$(echo "$eco_record" | jq -r '.repo') + detected=$(echo "$eco_record" | jq -r '.ecosystems | join(",")') + + dependabot_yml="$REPOS_DIR/$full_repo/.github/dependabot.yml" + dependabot_yaml="$REPOS_DIR/$full_repo/.github/dependabot.yaml" + + config_file="" + if [ -f "$dependabot_yml" ]; then + config_file="$dependabot_yml" + elif [ -f "$dependabot_yaml" ]; then + config_file="$dependabot_yaml" + fi + + if [ -z "$config_file" ]; then + # No dependabot config at all + if [ -n "$detected" ]; then + echo "$eco_record" | jq -c '. + {has_config: false, configured: [], gaps: .ecosystems}' \ + >> "$TMPDIR/coverage_gaps.jsonl" + fi + continue + fi + + # Extract configured ecosystems from dependabot.yml + # Map dependabot ecosystem names to our canonical names + configured="" + while IFS= read -r line; do + eco_raw=$(echo "$line" | sed -nE 's/.*package-ecosystem:[[:space:]]*"?([^"]*)"?.*/\1/p' | sed 's/[[:space:]]//g') + case "$eco_raw" in + pip) configured="${configured}pip," ;; + npm) configured="${configured}npm," ;; + gomod) configured="${configured}gomod," ;; + cargo) configured="${configured}cargo," ;; + docker) configured="${configured}docker," ;; + github-actions) configured="${configured}github-actions," ;; + esac + done < <(grep -E 'package-ecosystem:' "$config_file" 2>/dev/null || true) + configured="${configured%,}" + + # Find gaps: ecosystems detected but not configured + if [ -n "$detected" ]; then + gaps="" + IFS=',' read -ra detected_arr <<< "$detected" + for eco in "${detected_arr[@]}"; do + if ! echo ",$configured," | grep -qF ",$eco,"; then + gaps="${gaps}${eco}," + fi + done + gaps="${gaps%,}" + + if [ -n "$gaps" ]; then + jq -nc --arg repo "$full_repo" --arg detected "$detected" \ + --arg configured "$configured" --arg gaps "$gaps" \ + '{repo: $repo, has_config: true, + detected: ($detected | split(",")), + configured: ($configured | split(",") | map(select(. != ""))), + gaps: ($gaps | split(",") | map(select(. != "")))}' \ + >> "$TMPDIR/coverage_gaps.jsonl" + fi + fi +done < "$TMPDIR/ecosystems.jsonl" + +COVERAGE_GAPS=$(wc -l < "$TMPDIR/coverage_gaps.jsonl" | tr -d ' ') +echo "Repos with coverage gaps: $COVERAGE_GAPS" + +# --- Step 4: Filter stale PRs --- +jq -c 'select(.is_stale == true)' "$TMPDIR/all_prs.jsonl" > "$TMPDIR/stale_prs.jsonl" 2>/dev/null || true + +STALE_COUNT=$(wc -l < "$TMPDIR/stale_prs.jsonl" | tr -d ' ') +echo "" +echo "--- Stale PRs: $STALE_COUNT ---" + +# --- Step 5: Diff against previous scan --- +# Subset-run carry-forward (see docs/specs run-path, "Subset-aware stateful +# scanners"): when REPOMAN_SUBSET_REPOS is set, split the previous report into +# the stale PRs for SCANNED repos (diffed) and those for UNSCANNED repos +# (carried forward into latest.json, never diffed, so their issues are never +# mis-closed as resolved). +ALL_PREV_STALE="$TMPDIR/all_prev_stale.jsonl" +PREV_STALE="$TMPDIR/prev_stale.jsonl" +CARRY_STALE="$TMPDIR/carry_stale.jsonl" +: > "$CARRY_STALE" +if [ -f "$REPORTS_DIR/latest.json" ]; then + jq -c '.stale_prs[]?' "$REPORTS_DIR/latest.json" > "$ALL_PREV_STALE" 2>/dev/null || true +else + : > "$ALL_PREV_STALE" +fi +# Unscanned repos' coverage_gaps are also carried forward (they are per-.repo +# state that a subset run does not re-measure; dropping them would blank those +# repos' coverage until the next full run). +CARRY_COVERAGE="$TMPDIR/carry_coverage.jsonl" +: > "$CARRY_COVERAGE" +if [ -n "${REPOMAN_SUBSET_REPOS:-}" ]; then + SCANNED_JSON=$(printf '%s\n' "$REPOMAN_SUBSET_REPOS" \ + | jq -R 'select(length>0)' | jq -s '.' 2>/dev/null || echo '[]') + jq -c --argjson scanned "$SCANNED_JSON" \ + 'select(.repo as $r | $scanned | index($r))' \ + "$ALL_PREV_STALE" > "$PREV_STALE" 2>/dev/null || : > "$PREV_STALE" + jq -c --argjson scanned "$SCANNED_JSON" \ + 'select(.repo as $r | ($scanned | index($r)) | not)' \ + "$ALL_PREV_STALE" > "$CARRY_STALE" 2>/dev/null || : > "$CARRY_STALE" + # coverage_gaps for unscanned repos -> carried forward + if [ -f "$REPORTS_DIR/latest.json" ]; then + jq -c --argjson scanned "$SCANNED_JSON" \ + '.coverage_gaps[]? | select(.repo as $r | ($scanned | index($r)) | not)' \ + "$REPORTS_DIR/latest.json" > "$CARRY_COVERAGE" 2>/dev/null || : > "$CARRY_COVERAGE" + fi +else + cp "$ALL_PREV_STALE" "$PREV_STALE" +fi + +DIFF_KEY='[.repo, (.pr_number | tostring)] | join("|")' +read -r NEW_STALE FIXED_STALE RECURRING_STALE < <( + diff_against_previous "$TMPDIR/stale_prs.jsonl" "$PREV_STALE" "$DIFF_KEY" +) + +echo "Delta: +$NEW_STALE new, -$FIXED_STALE fixed, $RECURRING_STALE recurring" + +# --- Step 6: Create issues for NEW stale PRs --- +ISSUES_CREATED=0 + +while IFS='|' read -r issue_repo issue_pr_number; do + [ -z "$issue_repo" ] && continue + + # Check issue limit + if [ "$ISSUE_LIMIT" -gt 0 ] && [ "$ISSUES_CREATED" -ge "$ISSUE_LIMIT" ]; then + echo " SKIP (issue limit $ISSUE_LIMIT reached)" + break + fi + + # Get full record + record=$(jq -c "select(.repo == \"$issue_repo\" and .pr_number == ($issue_pr_number | tonumber))" \ + "$TMPDIR/stale_prs.jsonl" | head -1) + [ -z "$record" ] && continue + + # Extract fields + package=$(echo "$record" | jq -r '.package') + severity=$(echo "$record" | jq -r '.severity') + sla_days=$(echo "$record" | jq -r '.sla_days') + age_days=$(echo "$record" | jq -r '.age_days') + from_version=$(echo "$record" | jq -r '.from_version') + to_version=$(echo "$record" | jq -r '.to_version') + ecosystem=$(echo "$record" | jq -r '.ecosystem') + ci_status=$(echo "$record" | jq -r '.ci_status') + category=$(echo "$record" | jq -r '.category') + overdue=$((age_days - sla_days)) + + # issue_repo now carries full owner/name from Step 1 JSONL + full_repo="$issue_repo" + + # Deduplication + search_term="[dep-bump] Stale $severity bump: $package in $issue_repo" + existing=$(gh_issue_exists "$full_repo" "$search_term" || true) + + if [ -n "$existing" ]; then + echo " Issue #$existing already exists for $package in $issue_repo" + continue + fi + + # Build issue title (truncate if needed) + issue_title="[dep-bump] Stale $severity bump: $package in $issue_repo" + if [ ${#issue_title} -gt 250 ]; then + issue_title="${issue_title:0:247}..." + fi + + # Build version string + version_str="" + if [ -n "$from_version" ] && [ -n "$to_version" ]; then + version_str="$from_version -> $to_version" + else + version_str="(grouped update)" + fi + + # Build issue body + issue_body="## Describe the bug + +Stale Dependabot PR detected by automated dependency bump scan. + +**Repo:** $full_repo +**PR:** #$issue_pr_number +**Package:** $package +**Version:** $version_str +**Ecosystem:** $ecosystem +**Severity:** $severity +**SLA:** $sla_days days +**Age:** $age_days days (overdue by $overdue days) +**CI Status:** $ci_status +**First detected:** $SCAN_DATE +**Scan ID:** $SCAN_ID + +## Steps To Reproduce + +1. View PR: https://github.com/$full_repo/pull/$issue_pr_number +2. Note PR age ($age_days days) exceeds SLA threshold ($sla_days days) + +## Expected Behavior + +Dependabot PRs should be reviewed and merged within the SLA window. + +## Additional Context + +Category: $category + +--- +$SKILL_FOOTER (scan $SCAN_ID)" + + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would create issue: $issue_title" + ISSUES_CREATED=$((ISSUES_CREATED + 1)) + elif gh issue create --repo "$full_repo" \ + --title "$issue_title" \ + --body "$issue_body" 2>/dev/null; then + ISSUES_CREATED=$((ISSUES_CREATED + 1)) + echo " Created issue for $package in $issue_repo" + else + echo " WARN: Failed to create issue for $package in $issue_repo" + fi + + sleep 1 +done < "$TMPDIR/new_keys.txt" + +echo "Issues created: $ISSUES_CREATED" + +# --- Step 7: Close issues for FIXED (merged/closed PRs) --- +ISSUES_CLOSED=0 + +while IFS='|' read -r fix_repo fix_pr_number; do + [ -z "$fix_repo" ] && continue + + full_repo="$fix_repo" + + # Find matching open issue by searching for the PR number in title/body + issue_number=$(gh issue list --repo "$full_repo" \ + --search "[dep-bump] in:title #$fix_pr_number in:body" \ + --state open --json number --jq '.[0].number' 2>/dev/null || echo "") + + if [ -z "$issue_number" ] || [ "$issue_number" = "null" ]; then + continue + fi + + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would close issue #$issue_number (PR #$fix_pr_number merged/closed)" + ISSUES_CLOSED=$((ISSUES_CLOSED + 1)) + elif close_issue_if_valid "$full_repo" "$issue_number" \ + "Dependabot PR #$fix_pr_number has been merged/closed. Resolved in scan $SCAN_ID ($SCAN_DATE). Auto-closing."; then + ISSUES_CLOSED=$((ISSUES_CLOSED + 1)) + echo " Closed issue #$issue_number for PR #$fix_pr_number" + fi +done < "$TMPDIR/fixed_keys.txt" + +echo "Issues closed: $ISSUES_CLOSED" + +# --- Step 8: Write reports --- +# On a subset run, carry the unscanned repos' prior stale PRs forward so the +# baseline stays whole (they already carry their issue_number etc. verbatim). +STALE_ARRAY=$(jq -s '.' "$TMPDIR/stale_prs.jsonl" 2>/dev/null || echo "[]") +if [ -s "$CARRY_STALE" ]; then + CARRY_ARRAY=$(jq -s '.' "$CARRY_STALE" 2>/dev/null || echo "[]") + STALE_ARRAY=$(jq -n --argjson a "$STALE_ARRAY" --argjson b "$CARRY_ARRAY" '$a + $b' 2>/dev/null || printf '%s' "$STALE_ARRAY") +fi +COVERAGE_ARRAY=$(jq -s '.' "$TMPDIR/coverage_gaps.jsonl" 2>/dev/null || echo "[]") +if [ -s "$CARRY_COVERAGE" ]; then + CARRY_COV_ARRAY=$(jq -s '.' "$CARRY_COVERAGE" 2>/dev/null || echo "[]") + COVERAGE_ARRAY=$(jq -n --argjson a "$COVERAGE_ARRAY" --argjson b "$CARRY_COV_ARRAY" '$a + $b' 2>/dev/null || printf '%s' "$COVERAGE_ARRAY") +fi +ECO_SUMMARY=$(jq -s 'group_by(.ecosystem) | map({key: .[0].ecosystem, value: length}) | from_entries' \ + "$TMPDIR/all_prs.jsonl" 2>/dev/null || echo "{}") + +# Per-repo open-PR counts this run scanned (whole-set on a normal run). +REPO_PR_COUNTS=$(jq -s '.' "$TMPDIR/repo_pr_counts.jsonl" 2>/dev/null || echo "[]") + +# On a subset run the lists above are whole-set (carried forward), so the scalar +# population totals must be whole-set too, or the dashboard computes ratios +# across mixed scopes (e.g. subset total_open_prs vs whole-set stale_prs -> +# negative SLA). total_open_prs counts ALL open PRs, not just the stale ones in +# the lists, so it can't be derived from stale_prs/coverage_gaps. Instead we keep +# a per-repo open-PR breakdown (repo_pr_counts) and MERGE it: this run's counts +# for scanned repos + the prior report's counts for unscanned repos. The +# population totals are then recomputed from that merged breakdown, so they stay +# exactly consistent with the carried-forward lists. Self-pruning: the breakdown +# is rewritten each full run. +if [ -n "${REPOMAN_SUBSET_REPOS:-}" ] && [ -f "$REPORTS_DIR/latest.json" ]; then + SCANNED_JSON=$(printf '%s\n' "$REPOMAN_SUBSET_REPOS" \ + | jq -R 'select(length>0)' | jq -s '.' 2>/dev/null || echo '[]') + CARRY_PR_COUNTS=$(jq -c --argjson scanned "$SCANNED_JSON" \ + '[.repo_pr_counts[]? | select(.repo as $r | ($scanned | index($r)) | not)]' \ + "$REPORTS_DIR/latest.json" 2>/dev/null || echo '[]') + REPO_PR_COUNTS=$(jq -n --argjson a "$REPO_PR_COUNTS" --argjson b "$CARRY_PR_COUNTS" '$a + $b' 2>/dev/null || printf '%s' "$REPO_PR_COUNTS") + # recompute population totals from the merged per-repo breakdown + REPOS_WITH_DEPENDABOT=$(printf '%s' "$REPO_PR_COUNTS" | jq 'length') + TOTAL_OPEN_PRS=$(printf '%s' "$REPO_PR_COUNTS" | jq '[.[].open_prs] | add // 0') + # repos_scanned = distinct repos represented in the whole-set picture + REPOS_SCANNED=$(jq -r '.repos_scanned // '"$REPOS_SCANNED" "$REPORTS_DIR/latest.json" 2>/dev/null || echo "$REPOS_SCANNED") + # ecosystem_summary is whole-set in the prior report; carry it (subset can't recompute) + ECO_SUMMARY=$(jq -c '.ecosystem_summary // {}' "$REPORTS_DIR/latest.json" 2>/dev/null || echo "$ECO_SUMMARY") +fi + +LATEST_JSON=$(jq -nc \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson duration "$SECONDS" \ + --argjson repos_scanned "$REPOS_SCANNED" \ + --argjson repos_with_dependabot "$REPOS_WITH_DEPENDABOT" \ + --argjson total_open_prs "$TOTAL_OPEN_PRS" \ + --argjson stale_prs "$STALE_ARRAY" \ + --argjson coverage_gaps "$COVERAGE_ARRAY" \ + --argjson ecosystem_summary "$ECO_SUMMARY" \ + --argjson repo_pr_counts "$REPO_PR_COUNTS" \ + --argjson new_count "$NEW_STALE" \ + --argjson fixed_count "$FIXED_STALE" \ + --argjson recurring_count "$RECURRING_STALE" \ + '{scan_id: $scan_id, date: $date, duration_seconds: $duration, + repos_scanned: $repos_scanned, repos_with_dependabot: $repos_with_dependabot, + total_open_prs: $total_open_prs, stale_prs: $stale_prs, + coverage_gaps: $coverage_gaps, ecosystem_summary: $ecosystem_summary, + repo_pr_counts: $repo_pr_counts, + delta: {new: $new_count, fixed: $fixed_count, recurring: $recurring_count}}') + +# A dry run must NOT write the baseline (see link-health for the rationale): +# overwriting latest.json would make the next real run skip filing/closing. +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would write $REPORTS_DIR/latest.json (baseline left unchanged)" +else + write_report_latest "$REPORTS_DIR" "$LATEST_JSON" + echo "Wrote $REPORTS_DIR/latest.json" +fi + +# --- History row --- +STALE_SECURITY=$(jq -s '[.[] | select(.category == "security")] | length' "$TMPDIR/stale_prs.jsonl" 2>/dev/null || echo 0) +STALE_MAJOR=$(jq -s '[.[] | select(.category == "major")] | length' "$TMPDIR/stale_prs.jsonl" 2>/dev/null || echo 0) +STALE_ROUTINE=$(jq -s '[.[] | select(.category == "routine")] | length' "$TMPDIR/stale_prs.jsonl" 2>/dev/null || echo 0) + +HISTORY_ROW=$(jq -nc \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson repos_scanned "$REPOS_SCANNED" \ + --argjson total_open_prs "$TOTAL_OPEN_PRS" \ + --argjson stale_security "$STALE_SECURITY" \ + --argjson stale_major "$STALE_MAJOR" \ + --argjson stale_routine "$STALE_ROUTINE" \ + --argjson new_count "$NEW_STALE" \ + --argjson fixed_count "$FIXED_STALE" \ + --argjson issues_created "$ISSUES_CREATED" \ + --argjson issues_closed "$ISSUES_CLOSED" \ + '{scan_id: $scan_id, date: $date, + repos_scanned: $repos_scanned, total_open_prs: $total_open_prs, + stale_security: $stale_security, stale_major: $stale_major, + stale_routine: $stale_routine, + new: $new_count, fixed: $fixed_count, + issues_created: $issues_created, issues_closed: $issues_closed}') + +# Skip the history append on a subset run (rare manual recovery): history.json +# is a full-set trend series and a partial row would distort it. See +# docs/specs run-path, "Subset-aware stateful scanners". +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would append history.json row (skipped)" +elif [ -n "${REPOMAN_SUBSET_REPOS:-}" ]; then + echo "Subset run — skipping history.json append (trend series stays full-set)" +else + append_history_row "$REPORTS_DIR" "$HISTORY_ROW" "$MAX_HISTORY_ROWS" + echo "Appended to $REPORTS_DIR/history.json" +fi + +# Note: _index.json registration is performed by the run-path orchestrator +# (scripts/repoman-run.sh), the sole index writer — not here. Running this +# scanner directly no longer updates the registry; `repoman run dep-bump` is +# the blessed path for an indexed run. (Run path: issue #107, epic #72.) + +# --- Escalation check --- +if [ "$NEW_STALE" -gt "$ESCALATION_THRESHOLD" ]; then + echo "" + echo "ALERT: Dep bump scan found $NEW_STALE new stale PRs (threshold: $ESCALATION_THRESHOLD)." + echo "This may indicate a Dependabot wave or security advisory affecting many packages." + echo "Review Dependabot PRs in each enrolled repo's Security tab." +fi + +# --- Summary --- +echo "" +echo "=== Scan $SCAN_ID Summary ===" +echo "Repos: $REPOS_SCANNED scanned, $REPOS_WITH_DEPENDABOT with Dependabot PRs" +echo "PRs: $TOTAL_OPEN_PRS open, $STALE_COUNT stale ($STALE_SECURITY security, $STALE_MAJOR major, $STALE_ROUTINE routine)" +echo "Delta: +$NEW_STALE new, -$FIXED_STALE fixed, $RECURRING_STALE recurring" +echo "Coverage gaps: $COVERAGE_GAPS repos" +echo "Issues: $ISSUES_CREATED created, $ISSUES_CLOSED closed" +echo "Duration: ${SECONDS}s" diff --git a/skills/repoman/scripts/extract-broken-links.sh b/skills/repoman/scripts/extract-broken-links.sh new file mode 100755 index 0000000..ab1cbb0 --- /dev/null +++ b/skills/repoman/scripts/extract-broken-links.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +# extract-broken-links.sh — Turn lychee JSON output into broken-link records. +# +# Reads a lychee --format json report and emits one JSON object per broken +# link (JSONL, one record per line). URLs pointing at unreachable-by-design +# hostnames (Kubernetes cluster-local .svc names, *.local, RFC1918 ranges) are +# suppressed, since an external scanner can never reach them. lychee status is +# normalized to enum tokens: numeric HTTP codes stay as-is; text statuses map +# to timeout / dns / unreachable / error / unknown. +# +# This logic was previously inline in link-health-scanner.sh; it is extracted +# here so it can be unit-tested against synthetic lychee fixtures. +# +# Usage: +# extract-broken-links.sh +# cat report.json | extract-broken-links.sh - +# +# Arguments: +# Path to a lychee JSON report, or - to read from stdin. +# Full "owner/name" repo reference (e.g. "rossoctl/cortex"); +# emitted verbatim in the "repo" field. +# Absolute path prefix stripped from lychee's file keys +# (e.g. "/home/claw/kagenti/rossoctl/"). +# +# Output (stdout): JSONL, zero or more objects of the form +# {"repo":"rossoctl/foo","file":"docs/x.md","url":"https://...","status":"404","category":"external"} +# +# Exit codes: +# 0 - success (may emit zero records) +# 1 - usage error (missing arguments or file not found) + +# --- Argument handling --- +if [ $# -lt 3 ]; then + echo "Usage: extract-broken-links.sh " >&2 + echo " cat report.json | extract-broken-links.sh - " >&2 + exit 1 +fi + +LYCHEE_INPUT="$1" +REPO_FULL="$2" # full "owner/name" reference, emitted verbatim +REPOS_PREFIX="$3" + +if [ "$LYCHEE_INPUT" != "-" ] && [ ! -f "$LYCHEE_INPUT" ]; then + echo "ERROR: File not found: $LYCHEE_INPUT" >&2 + exit 1 +fi + +# Empty input produces no records. +if [ "$LYCHEE_INPUT" != "-" ] && [ ! -s "$LYCHEE_INPUT" ]; then + exit 0 +fi + +# --- Extract broken links from error_map (skip non-URL entries like "Error building URL") --- +# Normalize lychee status to enum tokens: numeric codes stay as-is, +# text statuses map to: timeout, dns, unreachable, error, unknown. +# Suppress URLs with unreachable-by-design hostnames (cluster-local, .local, RFC1918). +jq -r --arg repo "$REPO_FULL" --arg repos_dir "$REPOS_PREFIX" ' + .error_map // {} | to_entries[] | + .key as $filepath | + .value[] | + select(.url | test("^https?://")) | + # Suppress unreachable-by-design hostnames at URL level. + # Bare .svc covers both plain cluster-local service names (foo.ns.svc) and + # the fully-qualified foo.ns.svc.cluster.local form. + select(.url | test("://[^/]*\\.svc([:/]|$)") | not) | + select(.url | test("://[^/]*\\.svc\\.cluster\\.local([:/]|$)") | not) | + select(.url | test("://[^/]*\\.local([:/]|$)") | not) | + select(.url | test("://(10\\.[0-9]|172\\.(1[6-9]|2[0-9]|3[01])\\.[0-9]|192\\.168\\.[0-9])[0-9.]*([:/]|$)") | not) | + (.status.code // .status.text // null) as $raw_status | + ( + if $raw_status == null then "unknown" + elif ($raw_status | type) == "number" then ($raw_status | tostring) + elif ($raw_status | test("^[0-9]{3}$")) then $raw_status + elif ($raw_status | ascii_downcase | test("timeout")) then "timeout" + elif ($raw_status | ascii_downcase | test("resolve|dns")) then "dns" + elif ($raw_status | ascii_downcase | test("refused|reset|closed|unreachable|connect")) then "unreachable" + else "error" + end + ) as $status | + { + repo: $repo, + file: ($filepath | ltrimstr($repos_dir) | ltrimstr("./")), + url: .url, + status: $status, + category: ( + # internal = a link back into the scanned owner (derived from $repo, + # owner/name), so it follows the enrolled owner, not a hardcoded org list. + if (.url | test("github\\.com/" + ($repo | split("/")[0]) + "/")) then "internal" + else "external" + end + ) + } +' "${LYCHEE_INPUT/#-//dev/stdin}" diff --git a/skills/repoman/scripts/fork.sh b/skills/repoman/scripts/fork.sh new file mode 100644 index 0000000..5860e59 --- /dev/null +++ b/skills/repoman/scripts/fork.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# Fork/PR workflow helpers: fork creation, cross-fork PR creation, issue-field +# validation, and link-fix candidate scoring. +# +# ## Portability +# Targets bash 3.2+ (macOS default) through modern bash. +# +# No intra-library deps: every function here invokes gh/git/jq directly, so this +# module sources no sibling module. Self-contained for vendoring — copy it alone. +# If a function later calls a core.sh/github-api.sh helper, add a self-source +# block after the guard (resolved via ${BASH_SOURCE[0]}), per the design doc. +[ -n "${_FORK_SH_LOADED:-}" ] && return +_FORK_SH_LOADED=1 + +# ============================================================================= +# ISSUE BODY VALIDATION +# ============================================================================= + +# Validate fields extracted from a scanner-created issue body. +# Returns 0 if all fields pass validation, 1 if any field is malformed. +# This prevents injection via crafted issue bodies. +# +# Usage: +# if ! validate_issue_fields "$issue_repo" "$issue_file" "$broken_url" "$http_status" "$category"; then +# echo "WARN: Skipping issue with malformed fields" +# continue +# fi +# +# Args: +# $1 - repo field (expected: org/repo format) +# $2 - file field (expected: relative path, no ..) +# $3 - url field (expected: http:// or https://) +# $4 - status field (expected: numeric or known text) +# $5 - category field (expected: internal, external, or relative) +# Returns: 0 if valid, 1 if any field fails (prints warning to stderr) +validate_issue_fields() { + local repo="$1" + local file="$2" + local url="$3" + local status="$4" + local category="$5" + + # Repo must match org/repo pattern + if ! echo "$repo" | grep -qE '^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$'; then + echo " WARN: Invalid repo field: '$repo'" >&2 + return 1 + fi + + # File must not contain path traversal or shell metacharacters + if echo "$file" | grep -qE '(\.\./|[;$`|&<>])'; then + echo " WARN: Invalid file field (contains unsafe characters): '$file'" >&2 + return 1 + fi + + # URL must start with http:// or https:// + if ! echo "$url" | grep -qE '^https?://'; then + echo " WARN: Invalid URL field (not http/https): '$url'" >&2 + return 1 + fi + + # Status must be numeric or a known enum token + if ! echo "$status" | grep -qE '^([0-9]{3}|unknown|timeout|dns|unreachable|error)$'; then + echo " WARN: Invalid status field: '$status'" >&2 + return 1 + fi + + # Category must be a known value + if ! echo "$category" | grep -qE '^(internal|external|relative)$'; then + echo " WARN: Invalid category field: '$category'" >&2 + return 1 + fi + + return 0 +} + +# ============================================================================= +# PATH-SUFFIX SCORING +# ============================================================================= + +# Score a candidate path against a broken path by counting shared leading +# path segments (case-insensitive prefix match). This prioritizes candidates +# that share the most directory context with the broken URL, which best +# identifies where a renamed/moved file ended up. +# +# Example: +# score_path_suffix "authbridge/cmd/authbridge/README.md" "authbridge/cmd/README.md" +# # prints: 2 (shared prefix: authbridge/cmd) +# +# Args: +# $1 - broken path (the original path from the URL) +# $2 - candidate path (a found file) +# Prints: integer score (number of shared leading segments, 0 if none match) +score_path_suffix() { + local broken_path="$1" + local candidate_path="$2" + + # Lowercase both for case-insensitive comparison + local broken_lower candidate_lower + broken_lower=$(printf '%s' "$broken_path" | tr '[:upper:]' '[:lower:]') + candidate_lower=$(printf '%s' "$candidate_path" | tr '[:upper:]' '[:lower:]') + + # Split paths into arrays by "/" + IFS='/' read -ra broken_parts <<< "$broken_lower" + IFS='/' read -ra candidate_parts <<< "$candidate_lower" + + local broken_len=${#broken_parts[@]} + local candidate_len=${#candidate_parts[@]} + local max_compare=$((broken_len < candidate_len ? broken_len : candidate_len)) + local score=0 + + # Count shared leading segments + local i + for ((i = 0; i < max_compare; i++)); do + if [ "${broken_parts[$i]}" = "${candidate_parts[$i]}" ]; then + score=$((score + 1)) + else + break + fi + done + + echo "$score" +} + +# Given a broken path and a newline-separated list of candidates, find the +# best match by path-prefix score. Prints the winning candidate if there's +# a unique top scorer with score >= min_score; prints nothing otherwise. +# +# Usage: +# winner=$(pick_best_candidate "$target_path" "$candidates" 1) +# +# Args: +# $1 - broken path (from URL) +# $2 - newline-separated candidate paths +# $3 - minimum score threshold (default: 1) +# Prints: best candidate path if unique winner, empty if tied or below threshold +# Returns: 0 if unique winner found, 1 otherwise +pick_best_candidate() { + local broken_path="$1" + local candidates="$2" + local min_score="${3:-1}" + + local best_score=0 + local best_candidate="" + local tied=false + + while IFS= read -r candidate; do + [ -z "$candidate" ] && continue + local score + score=$(score_path_suffix "$broken_path" "$candidate") + + if [ "$score" -gt "$best_score" ]; then + best_score=$score + best_candidate="$candidate" + tied=false + elif [ "$score" -eq "$best_score" ] && [ "$score" -gt 0 ]; then + tied=true + fi + done <<< "$candidates" + + if [ "$best_score" -ge "$min_score" ] && [ "$tied" = false ] && [ -n "$best_candidate" ]; then + echo "$best_candidate" + return 0 + fi + + return 1 +} + +# ============================================================================= +# FORK / PR MANAGEMENT +# ============================================================================= + +# Ensure a fork of the target repo exists under the fork owner's account. +# If the fork doesn't exist, creates it and waits briefly for GitHub to +# propagate it (forks are not instantly available for push). +# +# Usage: +# ensure_fork "kagenti" "adk" "clawgenti" +# +# Args: +# $1 - org name (e.g., "kagenti") +# $2 - repo name (e.g., "adk") +# $3 - fork owner account (e.g., "clawgenti") +ensure_fork() { + local org="$1" + local repo_name="$2" + local fork_owner="$3" + + # Check if fork already exists + if gh repo view "$fork_owner/$repo_name" &>/dev/null 2>&1; then + return 0 + fi + + # Create the fork + gh repo fork "$org/$repo_name" --org "$fork_owner" --clone=false 2>/dev/null || true + + # Wait for GitHub to make the fork available for push + sleep 5 +} + +# Create a cross-fork PR from a fix branch. This function handles: +# 1. Adding the fork as a git remote (idempotent) +# 2. Committing staged changes with DCO sign-off +# 3. Pushing the branch to the fork +# 4. Creating the PR against upstream main +# +# Prerequisites: caller must have already cd'd into the repo directory, +# created the branch, and staged the changes (git add). +# +# Usage: +# pr_url=$(create_fork_pr "kagenti" "adk" "clawgenti" \ +# "fix/broken-links-2026-05-21" \ +# "docs: Fix broken link in README.md" \ +# "docs: Fix 1 broken internal link(s) in adk" \ +# "Automated fix by Kagenti Link Health Fixer.") +# +# Args: +# $1 - org name +# $2 - repo name +# $3 - fork owner +# $4 - branch name +# $5 - commit message +# $6 - PR title +# $7 - PR body +# Prints: the PR URL on success +# Returns: 0 on success, 1 on failure +create_fork_pr() { + local org="$1" + local repo_name="$2" + local fork_owner="$3" + local branch="$4" + local commit_msg="$5" + local pr_title="$6" + local pr_body="$7" + + local fork_remote="${fork_owner}-fork" + + # Add fork remote (idempotent -- silently skips if already exists) + if ! git remote get-url "$fork_remote" &>/dev/null 2>&1; then + git remote add "$fork_remote" \ + "https://github.com/$fork_owner/$repo_name.git" 2>/dev/null || true + fi + + # Commit with DCO sign-off (-s adds Signed-off-by trailer) + git commit -s -m "$commit_msg" + + # Push branch to the fork + if ! git push "$fork_remote" "$branch" --force-with-lease 2>/dev/null; then + echo " WARN: Failed to push branch $branch to $fork_owner/$repo_name" >&2 + return 1 + fi + + # Create cross-fork PR against upstream main + local pr_url + pr_url=$(gh pr create \ + --repo "$org/$repo_name" \ + --head "$fork_owner:$branch" \ + --base main \ + --title "$pr_title" \ + --body "$pr_body" \ + 2>/dev/null) + + if [ -z "$pr_url" ]; then + echo " WARN: Failed to create PR for $org/$repo_name" >&2 + return 1 + fi + + echo "$pr_url" +} diff --git a/skills/repoman/scripts/github-api.sh b/skills/repoman/scripts/github-api.sh new file mode 100644 index 0000000..9514e6a --- /dev/null +++ b/skills/repoman/scripts/github-api.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# GitHub API helpers: rate-limit-aware gh wrapper and issue read/close/PR-check +# operations. +# +# ## Portability +# Targets bash 3.2+ (macOS default) through modern bash. +# +# No intra-library deps: functions invoke gh/jq directly, so this module sources +# no sibling module. Self-contained for vendoring. +[ -n "${_GITHUB_API_SH_LOADED:-}" ] && return +_GITHUB_API_SH_LOADED=1 + +# ============================================================================= +# GITHUB API +# ============================================================================= + +# Rate-limit-aware wrapper around the gh CLI. +# Retries up to 3 times with exponential backoff on 403/429/rate-limit errors. +# +# Usage: +# gh_with_backoff issue list --repo kagenti/adk --state open --json number +# +# Global: increments RATE_LIMIT_BACKOFF counter (initialize to 0 before use) +RATE_LIMIT_BACKOFF=${RATE_LIMIT_BACKOFF:-0} + +gh_with_backoff() { + local attempt=0 + local max_attempts=3 + local wait=5 + while [ $attempt -lt $max_attempts ]; do + if output=$(gh "$@" 2>&1); then + RATE_LIMIT_BACKOFF=0 + printf '%s' "$output" + return 0 + fi + if echo "$output" | grep -qiE 'rate limit|403|429|secondary rate'; then + attempt=$((attempt + 1)) + RATE_LIMIT_BACKOFF=$((RATE_LIMIT_BACKOFF + 1)) + if [ $attempt -lt $max_attempts ]; then + echo " WARN: Rate limited, backing off ${wait}s (attempt $attempt/$max_attempts)" >&2 + sleep $wait + wait=$((wait * 2)) + fi + else + printf '%s' "$output" >&2 + return 1 + fi + done + echo " ERROR: Rate limit persisted after $max_attempts attempts, stopping" >&2 + return 1 +} + +# ============================================================================= +# GITHUB ISSUES +# ============================================================================= + +# Check if an open issue already exists that matches a search query. +# Used for deduplication before creating new issues. +# +# Usage: +# if existing=$(gh_issue_exists "kagenti/adk" "Broken link in README.md"); then +# echo "Issue #$existing already open" +# fi +# +# Args: +# $1 - full repo name (e.g., "kagenti/adk") +# $2 - search string (matched against issue title/body by GitHub) +# Returns: 0 if found (prints issue number), 1 if not found +gh_issue_exists() { + local repo="$1" + local search="$2" + local result + + result=$(gh issue list \ + --repo "$repo" \ + --search "$search" \ + --state open \ + --json number \ + --jq '.[0].number' \ + 2>/dev/null || echo "") + + if [ -n "$result" ] && [ "$result" != "null" ]; then + echo "$result" + return 0 + fi + + return 1 +} + +# Close an issue with a comment. Checks the exit code of `gh issue close` +# before reporting success -- this prevents the "close-before-verify" bug +# where we'd claim an issue was closed when the API call actually failed. +# +# Usage: +# if close_issue_if_valid "kagenti/adk" "123" "Fixed in scan 2026-05-21-001."; then +# echo "Closed" +# fi +# +# Args: +# $1 - full repo name +# $2 - issue number +# $3 - comment to add when closing +# Returns: 0 on success, 1 on failure (prints warning to stderr) +close_issue_if_valid() { + local repo="$1" + local number="$2" + local comment="$3" + + if gh issue close "$number" --repo "$repo" --comment "$comment" 2>/dev/null; then + return 0 + fi + + echo " WARN: Failed to close issue #$number in $repo" >&2 + return 1 +} + +# Returns 0 if any open PR in $repo plausibly covers $issue_number. +# Prints the covering PR number on stdout when found (for logging). +# +# Uses a three-layer detection strategy: +# Layer 1: GraphQL closingIssuesReferences (keyword-agnostic, author-agnostic) +# Layer 2: Keyword text search across all authors (close/fix/resolve variants) +# Layer 3: File + URL overlap in PR diffs (catches unlisted but covered issues) +# +# Args: +# $1 - full repo name (e.g., "kagenti/adk") +# $2 - issue number +# $3 - (optional) source file path from the issue body +# $4 - (optional) broken URL from the issue body +# Returns: 0 if covered, 1 otherwise +issue_has_open_pr() { + local repo="$1" + local issue_number="$2" + local source_file="${3:-}" + local broken_url="${4:-}" + local owner="${repo%/*}" + local repo_name="${repo#*/}" + local pr_number + + # Layer 1: GraphQL closingIssuesReferences (cheapest, most authoritative) + pr_number=$(gh api graphql -f query=' + query($owner:String!, $repo:String!, $num:Int!) { + repository(owner:$owner, name:$repo) { + issue(number:$num) { + closedByPullRequestsReferences(first:5, includeClosedPrs:false) { + nodes { number state } + } + } + } + }' \ + -F owner="$owner" -F repo="$repo_name" -F num="$issue_number" \ + --jq '.data.repository.issue.closedByPullRequestsReferences.nodes[] | select(.state == "OPEN") | .number' \ + 2>/dev/null | head -1) + + if [ -n "$pr_number" ]; then + echo "$pr_number" + return 0 + fi + + # Layer 2: Keyword search fallback (broader text match, all authors) + pr_number=$(gh pr list --repo "$repo" --state open \ + --search "#$issue_number" \ + --json number,body \ + --jq ".[] | select(.body | test(\"(?i)(close[sd]?|fix(e[sd])?|resolve[sd]?)\\\\s+#$issue_number(?![0-9])\")) | .number" \ + 2>/dev/null | head -1) + + if [ -n "$pr_number" ]; then + echo "$pr_number" + return 0 + fi + + # Layer 3: File + URL overlap (only when source_file and broken_url provided) + if [ -n "$source_file" ] && [ -n "$broken_url" ]; then + local pr_numbers + pr_numbers=$(gh pr list --repo "$repo" --state open \ + --json number,files \ + --jq ".[] | select(.files[]?.path == \"$source_file\") | .number" \ + 2>/dev/null) + + local candidate_pr + local escaped_url + escaped_url=$(printf '%s' "$broken_url" | sed -E 's/[.[\*^$()+?{|]/\\&/g') + + while IFS= read -r candidate_pr; do + [ -z "$candidate_pr" ] && continue + if gh pr diff "$candidate_pr" --repo "$repo" 2>/dev/null \ + | grep -Eq "^-.*$escaped_url"; then + echo "$candidate_pr" + return 0 + fi + done <<< "$pr_numbers" + fi + + return 1 +} + +# ============================================================================= +# SKILL PROVENANCE AND ATTRIBUTION (RepoMan Phase 6) +# ============================================================================= + +# Canonical skill source repo. Every attribution footer points here (the repo +# where skills are canonical), NOT at the automation repo that happens to host +# this library copy. +SKILL_SOURCE_REPO="${SKILL_SOURCE_REPO:-rossoctl/agent-skills}" + +# Resolve a skill's pinned commit SHA, writing a per-skill _meta file beside the +# skill on first success (create-only, self-healing). +# +# The meta file is keyed PER SKILL NAME (_meta..json), not per +# directory: in the flat scripts/ layout every program shares one SCRIPT_DIR, so +# a directory-only key would let whichever skill runs first pin ITS sha for all +# of them, and create-only means the wrong sha would never self-correct. Keying +# by name keeps each skill's pinned sha independent in any layout. +# +# On entry, if that per-skill meta file already exists it is read and its +# .version echoed -- no network call. Otherwise the SHA is resolved from the +# canonical source repo's commit history for that skill's path. On success the +# meta file is written and the SHA echoed; on failure (offline, unauthorized, +# rate limited) NOTHING is written -- the function warns to stderr and echoes +# empty, so the caller falls back to the blob/main attribution and the next run +# retries. It must never write a sentinel such as {"version":"unknown"}: a +# present-but-bogus file would stop the retry and pin a wrong version forever. +# +# Usage: sha=$(resolve_skill_meta "$SCRIPT_DIR" "link-health-scanner") +# Args: +# $1 - skill directory (where the meta file lives / will be written) +# $2 - skill name (its path segment under skills/ in the source repo) +# Prints: the resolved commit SHA, or empty string on failure. +# Returns: 0 always (provenance resolution never affects the scan/fix rc). +resolve_skill_meta() { + local skill_dir="$1" + local skill_name="$2" + local meta_file="$skill_dir/_meta.$skill_name.json" + local sha="" + + # Create-only: an existing _meta..json with a valid .version is + # authoritative, never overwritten (refreshing on skill update is the deploy + # layer's job). But a present-yet-corrupt file (truncated write, hand-edit, + # not JSON) yields no .version -- treating that as authoritative would pin the + # skill to the blob/main fallback forever with no self-heal. So we only short- + # circuit on a readable version; a corrupt file falls through to re-resolution, + # which overwrites it on success, honoring the spec's self-heal invariant. + if [ -f "$meta_file" ]; then + sha=$(jq -r '.version // empty' "$meta_file" 2>/dev/null) + if [ -n "$sha" ]; then + printf '%s' "$sha" + return 0 + fi + echo "WARN: $meta_file exists but carries no usable .version (corrupt?); re-resolving provenance." >&2 + fi + + # Resolve the pinned SHA from the canonical source repo's history for this + # skill's path. Failure here is non-fatal and leaves the meta file absent. + sha=$(gh api \ + "repos/$SKILL_SOURCE_REPO/commits?path=skills/$skill_name&per_page=1" \ + --jq '.[0].sha' 2>/dev/null) + + if [ -z "$sha" ] || [ "$sha" = "null" ]; then + echo "WARN: could not resolve provenance SHA for skill '$skill_name'; using blob/main attribution (will retry next run)." >&2 + printf '' + return 0 + fi + + # Write the per-skill _meta..json atomically beside the skill. + local installed_at + installed_at=$(date -u +"%Y-%m-%d") + local tmp="$meta_file.tmp.$$" + if jq -n \ + --arg version "$sha" \ + --arg source "$SKILL_SOURCE_REPO" \ + --arg installed_at "$installed_at" \ + '{version: $version, source: $source, installed_at: $installed_at}' \ + > "$tmp" 2>/dev/null && mv "$tmp" "$meta_file" 2>/dev/null; then + : + else + # Could not persist (read-only dir, etc.): still return the resolved SHA so + # this run gets a pinned footer, but leave no file so the next run retries. + rm -f "$tmp" 2>/dev/null + echo "WARN: resolved provenance SHA for '$skill_name' but could not write $meta_file (will retry next run)." >&2 + fi + + printf '%s' "$sha" + return 0 +} + +# Build the attribution footer for a skill's PR / issue body. +# +# Resolves the skill's pinned SHA (via resolve_skill_meta, using the caller's +# own SCRIPT_DIR as the skill directory so it works on every deploy layout) and +# returns the pinned-SHA RepoMan footer. On resolution failure it returns the +# blob/main fallback that pins to the live default branch instead -- additive, +# so a working scan never depends on a reachable GitHub API. +# +# Usage: footer=$(skill_attribution "link-health-scanner") +# Args: +# $1 - skill name (also the path segment under skills/) +# Prints: a one-line "Generated by RepoMan ..." attribution footer. +skill_attribution() { + local skill_name="$1" + local skill_dir="${SCRIPT_DIR:-.}" + local sha + sha=$(resolve_skill_meta "$skill_dir" "$skill_name") + + if [ -n "$sha" ]; then + printf 'Generated by RepoMan %s@%s (https://github.com/%s/blob/%s/skills/%s/SKILL.md).' \ + "$skill_name" "${sha:0:7}" "$SKILL_SOURCE_REPO" "$sha" "$skill_name" + else + printf 'Generated by RepoMan %s (https://github.com/%s/blob/main/skills/%s/SKILL.md).' \ + "$skill_name" "$SKILL_SOURCE_REPO" "$skill_name" + fi +} diff --git a/skills/repoman/scripts/link-health-fixer.sh b/skills/repoman/scripts/link-health-fixer.sh new file mode 100755 index 0000000..3c1a4bf --- /dev/null +++ b/skills/repoman/scripts/link-health-fixer.sh @@ -0,0 +1,701 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Link Health Fixer -- Phase 2 +# Reads scanner-created issues, re-verifies broken links, attempts fixes for internal links. +# Outputs PR previews in dry-run mode; creates fork-based PRs when --live is passed. + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- Configuration --- +REPORTS_DIR="${REPORTS_DIR:-$HOME/reports/link-health}" + +# Provenance footer for fix-PR bodies. Resolved once at startup: the helper +# lazily writes _meta..json beside this skill and returns the pinned-SHA RepoMan +# attribution (blob/main fallback when the SHA cannot be resolved). +SKILL_FOOTER=$(skill_attribution "link-health-fixer") + +FIX_DATE=$(date +%Y-%m-%d) +SCAN_DATE=$(date +%Y-%m-%d) + +# DCO sign-off identity for fix PRs (maintainer, not bot) +GIT_AUTHOR_NAME="Gloire Rubambiza" +GIT_AUTHOR_EMAIL="gloire@ibm.com" +export GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL +export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" +export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" + +# --- CLI args --- +DRY_RUN=true +ISSUE_LIMIT=5 +VERBOSE=false + +while [[ $# -gt 0 ]]; do + case $1 in + --live) DRY_RUN=false; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --issue-limit) ISSUE_LIMIT="$2"; shift 2 ;; + --verbose) VERBOSE=true; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Resolve deployment constants (repos_dir, fork_owner) from ~/.repoman/config.json. +repoman_config + +# --- Workspace setup --- +setup_workspace "link-fixer" +TMPDIR="$PROGRAM_TMPDIR" + +echo "=== Link Health Fixer $FIX_DATE ===" +echo "Repos dir: $REPOS_DIR" +if [ "$DRY_RUN" = true ]; then + echo "Mode: DRY RUN (no PRs, re-verified issues will still be closed)" +else + echo "Mode: LIVE (PRs will be created)" +fi +echo "Issue limit: $ISSUE_LIMIT" +echo "" + +# --- Step 1: Gather open scanner issues across all repos --- +echo "=== Step 1: Gathering open scanner issues ===" + +ISSUES_FILE="$TMPDIR/issues.jsonl" +: > "$ISSUES_FILE" + +# Drive the loop from the enrolled set (authoritative), statting each clone, +# rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently +# drops a ".github" repo. enrolled_clone_dirs emits each enrolled owner/name +# whose clone exists; the here-string keeps the body in the current shell. +ENROLLED=$(repoman_load_enrolled) || exit 1 +CLONES=$(enrolled_clone_dirs "$ENROLLED") + +while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + + issues_json=$(gh issue list --repo "$full_repo" \ + --search "Broken link in:title" \ + --state open --limit 100 \ + --json number,title,body 2>/dev/null || echo "[]") + + # Add repository info since gh issue list doesn't include it + echo "$issues_json" | jq -c --arg repo "$full_repo" \ + '.[] | . + {repository: {nameWithOwner: $repo}}' >> "$ISSUES_FILE" 2>/dev/null || true +done <<< "$CLONES" + +TOTAL_ISSUES=$(wc -l < "$ISSUES_FILE" | tr -d ' ') +echo "Found $TOTAL_ISSUES open scanner issues" + +if [ "$TOTAL_ISSUES" -eq 0 ]; then + echo "No issues to process." + echo "" + echo "=== Fixer Run $FIX_DATE Summary ===" + echo "Issues processed: 0" + exit 0 +fi + +# --- Step 2: Parse issue fields and filter to internal links --- +echo "" +echo "=== Step 2: Parsing issues and classifying ===" + +PARSED_FILE="$TMPDIR/parsed.jsonl" +EXTERNAL_FILE="$TMPDIR/external.jsonl" +: > "$PARSED_FILE" +: > "$EXTERNAL_FILE" + +while IFS= read -r issue_json; do + number=$(echo "$issue_json" | jq -r '.number') + repo_full=$(echo "$issue_json" | jq -r '.repository.nameWithOwner // empty') + body=$(echo "$issue_json" | jq -r '.body // ""') + + # Parse structured fields from issue body (BSD-compatible, no grep -P) + issue_repo=$(echo "$body" | sed -nE 's/^\*\*Repo:\*\*[[:space:]]*(.*)/\1/p' | head -1 | tr -d ' ' || true) + issue_file=$(echo "$body" | sed -nE 's/^\*\*File:\*\*[[:space:]]*(.*)/\1/p' | head -1 | tr -d ' ' || true) + broken_url=$(echo "$body" | sed -nE 's/^\*\*Broken URL:\*\*[[:space:]]*(.*)/\1/p' | head -1 | tr -d ' ' || true) + http_status=$(echo "$body" | sed -nE 's/^\*\*HTTP Status:\*\*[[:space:]]*(.*)/\1/p' | head -1 || true) + category=$(echo "$body" | sed -nE 's/^Category:[[:space:]]*(.*)/\1/p' | head -1 | tr -d ' ' || true) + + # Skip issues with missing required fields + [ -z "$issue_repo" ] && continue + [ -z "$broken_url" ] && continue + + # Validate extracted fields to prevent injection via crafted issue bodies + if ! validate_issue_fields "$issue_repo" "${issue_file:-unknown}" "$broken_url" "${http_status:-unknown}" "${category:-internal}"; then + echo " WARN: Skipping issue #$number with malformed fields" + continue + fi + + # Use repo from issue body if repo_full is empty + [ -z "$repo_full" ] && repo_full="$issue_repo" + + record=$(jq -nc \ + --arg number "$number" \ + --arg repo "$repo_full" \ + --arg issue_repo "$issue_repo" \ + --arg file "$issue_file" \ + --arg url "$broken_url" \ + --arg status "$http_status" \ + --arg category "$category" \ + '{number: ($number|tonumber), repo: $repo, issue_repo: $issue_repo, file: $file, url: $url, status: $status, category: $category}') + + if [ "$category" = "internal" ]; then + echo "$record" >> "$PARSED_FILE" + elif [ "$category" = "external" ]; then + echo "$record" >> "$EXTERNAL_FILE" + else + [ "$VERBOSE" = true ] && echo " SKIP #$number: unknown category ($category)" + fi +done < "$ISSUES_FILE" + +INTERNAL_COUNT=$(wc -l < "$PARSED_FILE" | tr -d ' ') +EXTERNAL_COUNT=$(wc -l < "$EXTERNAL_FILE" | tr -d ' ') +echo "Internal link issues: $INTERNAL_COUNT" +echo "External link issues: $EXTERNAL_COUNT" +echo "Total: $TOTAL_ISSUES" + +# --- Step 3: Re-verify broken links --- +echo "" +echo "=== Step 3: Re-verifying broken links ===" + +STILL_BROKEN="$TMPDIR/still_broken.jsonl" +REVERIFIED="$TMPDIR/reverified.jsonl" +: > "$STILL_BROKEN" +: > "$REVERIFIED" + +PROCESSED=0 +REVERIFIED_COUNT=0 + +while IFS= read -r item; do + [ "$ISSUE_LIMIT" -gt 0 ] && [ "$PROCESSED" -ge "$ISSUE_LIMIT" ] && break + + number=$(echo "$item" | jq -r '.number') + repo=$(echo "$item" | jq -r '.repo') + broken_url=$(echo "$item" | jq -r '.url') + source_file=$(echo "$item" | jq -r '.file') + + # Skip issues that already have an open fix PR (don't count toward limit) + if covering_pr=$(issue_has_open_pr "$repo" "$number" "$source_file" "$broken_url"); then + echo " #$number: Open PR #$covering_pr already covers, skipping" + continue + fi + + # Extract target org/repo/path from GitHub URL (BSD-compatible) + if echo "$broken_url" | grep -qE 'github\.com/[^/]+/[^/]+/(blob|tree)/'; then + target_org=$(echo "$broken_url" | sed -nE 's#.*github\.com/([^/]+)/.*#\1#p') + target_repo=$(echo "$broken_url" | sed -nE 's#.*github\.com/[^/]+/([^/]+)/.*#\1#p') + target_ref=$(echo "$broken_url" | sed -nE 's#.*(blob|tree)/([^/]+)/.*#\2#p') + target_path=$(echo "$broken_url" | sed -E 's#.*/((blob|tree))/[^/]+/##') + + # Check if file exists via GitHub API + if gh api "repos/$target_org/$target_repo/contents/$target_path?ref=$target_ref" \ + --jq '.name' >/dev/null 2>&1; then + echo " #$number: Link now resolves ($broken_url)" + + if close_issue_if_valid "$repo" "$number" \ + "Link now resolves as of $SCAN_DATE. Verified by Link Health Fixer. Closing."; then + echo " #$number: Issue closed" + fi + + echo "$item" >> "$REVERIFIED" + REVERIFIED_COUNT=$((REVERIFIED_COUNT + 1)) + PROCESSED=$((PROCESSED + 1)) + sleep 1 + continue + fi + fi + + echo " #$number: Still broken ($broken_url)" + echo "$item" >> "$STILL_BROKEN" + PROCESSED=$((PROCESSED + 1)) +done < "$PARSED_FILE" + +BROKEN_COUNT=$(wc -l < "$STILL_BROKEN" | tr -d ' ') +echo "Re-verified (now valid): $REVERIFIED_COUNT" +echo "Still broken: $BROKEN_COUNT" + +# --- Step 4: Attempt deterministic fixes --- +echo "" +echo "=== Step 4: Attempting fixes ===" + +FIXES_FILE="$TMPDIR/fixes.jsonl" +AMBIGUOUS_FILE="$REPORTS_DIR/fixer-ambiguous.json" +: > "$FIXES_FILE" +echo "[]" > "$AMBIGUOUS_FILE" + +FIXED_COUNT=0 +UNFIXABLE_COUNT=0 +AMBIGUOUS_COUNT=0 + +while IFS= read -r item; do + number=$(echo "$item" | jq -r '.number') + repo=$(echo "$item" | jq -r '.repo') + issue_repo=$(echo "$item" | jq -r '.issue_repo') + source_file=$(echo "$item" | jq -r '.file') + broken_url=$(echo "$item" | jq -r '.url') + + # Skip issues that already have an open fix PR + if covering_pr=$(issue_has_open_pr "$repo" "$number" "$source_file" "$broken_url"); then + echo " #$number: Open PR #$covering_pr already covers, skipping" + continue + fi + + # Extract target owner, repo, and path from URL + target_org="" + target_repo_name="" + target_path="" + + if echo "$broken_url" | grep -qE 'github\.com/[^/]+/[^/]+/(blob|tree)/'; then + target_org=$(echo "$broken_url" | sed -nE 's#.*github\.com/([^/]+)/.*#\1#p') + target_repo_name=$(echo "$broken_url" | sed -nE 's#.*github\.com/[^/]+/([^/]+)/.*#\1#p') + target_path=$(echo "$broken_url" | sed -E 's#.*/((blob|tree))/[^/]+/##') + else + echo " #$number: Unrecognized URL format, skipping" + UNFIXABLE_COUNT=$((UNFIXABLE_COUNT + 1)) + continue + fi + + # Owner-namespaced clone layout: the target clone lives at + # $REPOS_DIR//, so the owner parsed from the URL is required + # to locate it. A bare-name join would miss it and mark the link unfixable. + target_repo_dir="$REPOS_DIR/$target_org/$target_repo_name" + if [ ! -d "$target_repo_dir/.git" ]; then + echo " #$number: Target repo $target_org/$target_repo_name not cloned locally, skipping" + UNFIXABLE_COUNT=$((UNFIXABLE_COUNT + 1)) + continue + fi + + echo " #$number: Searching for $target_path in $target_repo_name..." + + # Strategy 1: git log --follow for renames + filename=$(basename "$target_path") + new_path="" + + rename_result=$(cd "$target_repo_dir" && git log --all --follow --diff-filter=R \ + --format="%H" -- "$target_path" 2>/dev/null | head -1 || true) + + if [ -n "$rename_result" ]; then + # Find what the file was renamed to + new_path=$(cd "$target_repo_dir" && git diff-tree --no-commit-id -r --diff-filter=R \ + "$rename_result" 2>/dev/null | grep "$filename" | awk '{print $NF}' | head -1 || true) + [ -n "$new_path" ] && [ -f "$target_repo_dir/$new_path" ] || new_path="" + fi + + # Strategy 2: find by filename + if [ -z "$new_path" ]; then + candidates=$(cd "$target_repo_dir" && find . -name "$filename" -type f \ + ! -path '*/node_modules/*' ! -path '*/.git/*' ! -path '*/vendor/*' \ + 2>/dev/null | sed 's|^\./||') + + candidate_count=$(echo "$candidates" | grep -c . 2>/dev/null || echo 0) + + if [ "$candidate_count" -eq 1 ] && [ -n "$candidates" ]; then + new_path="$candidates" + elif [ "$candidate_count" -gt 1 ]; then + # Score candidates by path-prefix match + best=$(pick_best_candidate "$target_path" "$candidates" 1) || true + if [ -n "$best" ]; then + echo " Scored best candidate: $best" + new_path="$best" + else + echo " Multiple candidates found ($candidate_count), flagging for model" + candidate_list=$(echo "$candidates" | jq -R . | jq -s .) + jq --argjson new "$(jq -nc \ + --arg number "$number" \ + --arg repo "$repo" \ + --arg issue_repo "$issue_repo" \ + --arg file "$source_file" \ + --arg url "$broken_url" \ + --arg target_repo "$target_repo_name" \ + --arg target_path "$target_path" \ + --argjson candidates "$candidate_list" \ + '{issue_number: ($number|tonumber), repo: $repo, file: $file, broken_url: $url, target_repo: $target_repo, target_path: $target_path, candidates: $candidates, reason: "multiple_matches"}')" \ + '. += [$new]' "$AMBIGUOUS_FILE" > "$TMPDIR/amb_tmp.json" && mv "$TMPDIR/amb_tmp.json" "$AMBIGUOUS_FILE" + AMBIGUOUS_COUNT=$((AMBIGUOUS_COUNT + 1)) + continue + fi + fi + fi + + # Strategy 3: find by filename without extension (for .mdx -> .md or vice versa) + if [ -z "$new_path" ]; then + basename_no_ext="${filename%.*}" + candidates=$(cd "$target_repo_dir" && find . -name "${basename_no_ext}.*" -type f \ + ! -path '*/node_modules/*' ! -path '*/.git/*' ! -path '*/vendor/*' \ + 2>/dev/null | sed 's|^\./||') + + candidate_count=$(echo "$candidates" | grep -c . 2>/dev/null || echo 0) + + if [ "$candidate_count" -eq 1 ] && [ -n "$candidates" ]; then + new_path="$candidates" + elif [ "$candidate_count" -gt 1 ]; then + # Score candidates by path-prefix match + best=$(pick_best_candidate "$target_path" "$candidates" 1) || true + if [ -n "$best" ]; then + echo " Scored best candidate: $best" + new_path="$best" + else + echo " Multiple extension variants found ($candidate_count), flagging for model" + candidate_list=$(echo "$candidates" | jq -R . | jq -s .) + jq --argjson new "$(jq -nc \ + --arg number "$number" \ + --arg repo "$repo" \ + --arg issue_repo "$issue_repo" \ + --arg file "$source_file" \ + --arg url "$broken_url" \ + --arg target_repo "$target_repo_name" \ + --arg target_path "$target_path" \ + --argjson candidates "$candidate_list" \ + '{issue_number: ($number|tonumber), repo: $repo, file: $file, broken_url: $url, target_repo: $target_repo, target_path: $target_path, candidates: $candidates, reason: "multiple_extension_matches"}')" \ + '. += [$new]' "$AMBIGUOUS_FILE" > "$TMPDIR/amb_tmp.json" && mv "$TMPDIR/amb_tmp.json" "$AMBIGUOUS_FILE" + AMBIGUOUS_COUNT=$((AMBIGUOUS_COUNT + 1)) + continue + fi + fi + fi + + if [ -z "$new_path" ]; then + echo " No candidates found, marking unfixable" + UNFIXABLE_COUNT=$((UNFIXABLE_COUNT + 1)) + + # Comment on the issue + gh issue comment "$number" --repo "$repo" \ + --body "Unable to find an automated fix: file \`$target_path\` not found in \`$target_repo_name\` (searched for renames, filename matches, and extension variants). This may require manual investigation." \ + 2>/dev/null || echo " WARN: Failed to comment on issue #$number" + sleep 1 + continue + fi + + # Build new URL (BSD-compatible) + target_ref=$(echo "$broken_url" | sed -nE 's#.*(blob|tree)/([^/]+)/.*#\2#p') + url_type=$(echo "$broken_url" | sed -nE 's#.*(blob|tree)/.*#\1#p') + # target_org was already parsed from $broken_url above. + new_url="https://github.com/$target_org/$target_repo_name/$url_type/$target_ref/$new_path" + + # Verify the new path exists + if [ ! -f "$target_repo_dir/$new_path" ]; then + echo " Candidate $new_path does not exist on disk, skipping" + UNFIXABLE_COUNT=$((UNFIXABLE_COUNT + 1)) + continue + fi + + echo " FIX: $target_path -> $new_path" + + # $repo is already the full "owner/name" (from the enrolled-repo scan loop), + # so it carries the full ref straight through to .source_repo -- which is + # grouped and compared as a full ref downstream (Step 5). No alias: passing a + # separate bare-vs-full-named copy is exactly the shape mismatch to avoid. + jq -nc \ + --arg number "$number" \ + --arg repo "$repo" \ + --arg source_repo "$repo" \ + --arg source_file "$source_file" \ + --arg old_url "$broken_url" \ + --arg new_url "$new_url" \ + --arg old_path "$target_path" \ + --arg new_path "$new_path" \ + '{number: ($number|tonumber), repo: $repo, source_repo: $source_repo, source_file: $source_file, old_url: $old_url, new_url: $new_url, old_path: $old_path, new_path: $new_path}' \ + >> "$FIXES_FILE" + + FIXED_COUNT=$((FIXED_COUNT + 1)) +done < "$STILL_BROKEN" + +echo "" +echo "Fixes found: $FIXED_COUNT" +echo "Ambiguous (needs model): $AMBIGUOUS_COUNT" +echo "Unfixable: $UNFIXABLE_COUNT" + +# --- Step 5: Apply fixes / PR preview --- +echo "" +echo "=== Step 5: Fix application ===" + +if [ "$FIXED_COUNT" -eq 0 ]; then + echo "No fixes to apply." +else + # Group fixes by source repo + repos_with_fixes=$(jq -r '.source_repo' "$FIXES_FILE" | sort -u) + + for fix_repo in $repos_with_fixes; do + echo "" + echo "--- Repo: $fix_repo ---" + + # fix_repo is "owner/name" (from the enrolled-repo scan loop). Derive the + # bare name for filesystem/branch/remote tokens that must not contain "/". + fix_repo_name="${fix_repo#*/}" + + fix_repo_dir="$REPOS_DIR/$fix_repo" + FIX_BRANCH="fix/broken-links-${fix_repo_name}-${FIX_DATE}" + + # Collect fixes for this repo + repo_fixes=$(jq -c "select(.source_repo == \"$fix_repo\")" "$FIXES_FILE") + fix_count=$(echo "$repo_fixes" | wc -l | tr -d ' ') + echo "Fixes: $fix_count" + + if [ "$DRY_RUN" = true ]; then + echo "" + echo "[DRY RUN] PR Preview for $fix_repo:" + echo " Branch: $FIX_BRANCH" + echo " Title: docs: Fix $fix_count broken internal link(s) in $fix_repo_name" + echo " Changes:" + + echo "$repo_fixes" | while IFS= read -r fix; do + src_file=$(echo "$fix" | jq -r '.source_file') + old_url=$(echo "$fix" | jq -r '.old_url') + new_url=$(echo "$fix" | jq -r '.new_url') + echo " $src_file:" + echo " - $old_url" + echo " + $new_url" + done + + echo "" + echo " Body:" + echo " Automated fix by RepoMan Link Health Fixer." + echo " Broken internal links updated to point to current file locations." + echo "" + echo " | File | Old URL | New URL |" + echo " |------|---------|---------|" + echo "$repo_fixes" | while IFS= read -r fix; do + src_file=$(echo "$fix" | jq -r '.source_file') + old_path=$(echo "$fix" | jq -r '.old_path') + new_path=$(echo "$fix" | jq -r '.new_path') + echo " | \`$src_file\` | \`$old_path\` | \`$new_path\` |" + done + else + # --- LIVE MODE --- + cd "$fix_repo_dir" + + # Fork on demand (uses shared library) + ensure_fork "${fix_repo%%/*}" "${fix_repo#*/}" "$FORK_OWNER" + + # Set up remote + FORK_REMOTE="clawgenti-${fix_repo_name}-fork" + if ! git remote get-url "$FORK_REMOTE" &>/dev/null 2>&1; then + git remote add "$FORK_REMOTE" "https://github.com/$FORK_OWNER/${fix_repo_name}.git" + fi + + # Create branch from fork's main + git fetch "$FORK_REMOTE" main 2>/dev/null || true + git checkout -B "$FIX_BRANCH" "$FORK_REMOTE/main" 2>/dev/null || git checkout -B "$FIX_BRANCH" + + # Apply each fix + echo "$repo_fixes" | while IFS= read -r fix; do + src_file=$(echo "$fix" | jq -r '.source_file') + old_url=$(echo "$fix" | jq -r '.old_url') + new_url=$(echo "$fix" | jq -r '.new_url') + + if [ -f "$src_file" ]; then + # Escape URLs for sed + old_escaped=$(printf '%s\n' "$old_url" | sed 's/[&/\]/\\&/g') + new_escaped=$(printf '%s\n' "$new_url" | sed 's/[&/\]/\\&/g') + sed -i "s|$old_escaped|$new_escaped|g" "$src_file" + git add "$src_file" + else + echo " WARN: Source file $src_file not found" + fi + done + + # Commit + git commit -s -m "docs: Fix broken internal links in $fix_repo_name + +Automated fix by RepoMan Link Health Fixer ($FIX_DATE)." 2>/dev/null || { + echo " No changes to commit" + continue + } + + # Push + git push "$FORK_REMOTE" "$FIX_BRANCH" 2>/dev/null || { + echo " WARN: Failed to push to fork" + continue + } + + # Create PR + pr_body="Automated fix by RepoMan Link Health Fixer. +Broken internal links updated to point to current file locations. + +| File | Old Path | New Path | +|------|----------|----------|" + + while IFS= read -r fix; do + src_file=$(echo "$fix" | jq -r '.source_file') + old_path=$(echo "$fix" | jq -r '.old_path') + new_path=$(echo "$fix" | jq -r '.new_path') + pr_body="$pr_body +| \`$src_file\` | \`$old_path\` | \`$new_path\` |" + done <<< "$repo_fixes" + + # Breadcrumb footer linking back to the program's canonical skill, so a + # reader on the target repo can find what generated this PR. Mirrors the + # scanner's footer (link-health-scanner.sh). + pr_body="$pr_body + +$SKILL_FOOTER" + + pr_url=$(gh pr create --repo "$fix_repo" \ + --head "$FORK_OWNER:$FIX_BRANCH" --base main \ + --title "docs: Fix $fix_count broken internal link(s) in $fix_repo_name" \ + --body "$pr_body" 2>/dev/null) || { + echo " WARN: Failed to create PR" + continue + } + + echo " PR created: $pr_url" + + # Comment on each issue with the PR link + echo "$repo_fixes" | while IFS= read -r fix; do + issue_num=$(echo "$fix" | jq -r '.number') + src_file=$(echo "$fix" | jq -r '.source_file') + new_path=$(echo "$fix" | jq -r '.new_path') + gh issue comment "$issue_num" --repo "$fix_repo" \ + --body "Fix submitted: $pr_url. The broken link in \`$src_file\` has been updated to point to \`$new_path\`." \ + 2>/dev/null || echo " WARN: Failed to comment on issue #$issue_num" + sleep 1 + done + + # Return to main + git checkout main 2>/dev/null || true + fi + done +fi + +# --- Step 6: Analyze external links --- +echo "" +echo "=== Step 6: Analyzing external links ===" + +EXTERNAL_ANALYZED=0 +EXTERNAL_REVERIFIED=0 +EXTERNAL_PROCESSED=0 + +while IFS= read -r item; do + [ "$ISSUE_LIMIT" -gt 0 ] && [ "$((PROCESSED + EXTERNAL_PROCESSED))" -ge "$ISSUE_LIMIT" ] && break + + number=$(echo "$item" | jq -r '.number') + repo=$(echo "$item" | jq -r '.repo') + broken_url=$(echo "$item" | jq -r '.url') + source_file=$(echo "$item" | jq -r '.file') + + # Skip issues that already have an open fix PR (don't count toward limit) + if covering_pr=$(issue_has_open_pr "$repo" "$number" "$source_file" "$broken_url"); then + echo " #$number: Open PR #$covering_pr already covers, skipping" + continue + fi + + echo " #$number: Analyzing $broken_url" + + # Re-verify with full redirect follow + curl_output=$(curl -sL -o /dev/null -w "%{http_code} %{url_effective}" --max-time 10 "$broken_url" 2>/dev/null || echo "000 ") + final_status=$(echo "$curl_output" | awk '{print $1}') + final_url=$(echo "$curl_output" | awk '{print $2}') + + if [ "$final_status" = "200" ]; then + if [ "$final_url" = "$broken_url" ]; then + # Link resolves at same URL -- close the issue + echo " Link now resolves (200 at original URL)" + if close_issue_if_valid "$repo" "$number" \ + "Link now resolves as of $FIX_DATE. Verified by Link Health Fixer. Closing."; then + echo " Issue closed" + fi + EXTERNAL_REVERIFIED=$((EXTERNAL_REVERIFIED + 1)) + EXTERNAL_PROCESSED=$((EXTERNAL_PROCESSED + 1)) + sleep 1 + continue + else + # Link moved -- report replacement but don't close + echo " Link moved (200 at different URL): $final_url" + redirect_finding="Redirects to: \`$final_url\`" + suggested_replacement="\`$final_url\` -- this is where the original URL now redirects." + fi + else + echo " Still broken (HTTP $final_status)" + redirect_finding="No redirect detected (HTTP $final_status)" + suggested_replacement="" + fi + + # Check Wayback Machine + wayback_url="" + wayback_date="" + wayback_json=$(curl -s --max-time 10 "https://web.archive.org/wayback/available?url=$broken_url" 2>/dev/null || echo "{}") + wayback_url=$(echo "$wayback_json" | jq -r '.archived_snapshots.closest.url // empty' 2>/dev/null || true) + wayback_date=$(echo "$wayback_json" | jq -r '.archived_snapshots.closest.timestamp // empty' 2>/dev/null || true) + + if [ -n "$wayback_url" ]; then + # Format timestamp (YYYYMMDDHHMMSS -> YYYY-MM-DD) + wb_formatted="${wayback_date:0:4}-${wayback_date:4:2}-${wayback_date:6:2}" + wayback_finding="Archived version available from $wb_formatted: $wayback_url" + if [ -z "$suggested_replacement" ]; then + suggested_replacement="\`$wayback_url\` -- archived version from $wb_formatted." + fi + else + wayback_finding="No archived version found" + fi + + if [ -z "$suggested_replacement" ]; then + suggested_replacement="No automated replacement found. Manual investigation needed." + fi + + # Build comment + comment_body="## External Link Analysis + +**URL:** \`$broken_url\` +**Current status:** HTTP $final_status +**Source file:** \`$source_file\` + +### Findings +- **Redirect:** $redirect_finding +- **Wayback Machine:** $wayback_finding + +### Suggested replacement +$suggested_replacement + +--- +*Analyzed by RepoMan Link Health Fixer ($FIX_DATE)*" + + # Post comment + if gh issue comment "$number" --repo "$repo" --body "$comment_body" 2>/dev/null; then + echo " Comment posted" + else + echo " WARN: Failed to comment on issue #$number" + fi + + # Label unfixable if no redirect and no wayback + if [ "$final_status" != "200" ] && [ -z "$wayback_url" ]; then + gh issue edit "$number" --repo "$repo" --add-label "broken-link/unfixable" 2>/dev/null || true + fi + + EXTERNAL_ANALYZED=$((EXTERNAL_ANALYZED + 1)) + EXTERNAL_PROCESSED=$((EXTERNAL_PROCESSED + 1)) + sleep 1 +done < "$EXTERNAL_FILE" + +echo "External analyzed: $EXTERNAL_ANALYZED" +echo "External re-verified (now valid): $EXTERNAL_REVERIFIED" + +# --- Step 7: Ambiguous items for model --- +AMBIG_COUNT=$(jq 'length' "$AMBIGUOUS_FILE") +if [ "$AMBIG_COUNT" -gt 0 ]; then + echo "" + echo "=== Ambiguous items (needs model reasoning) ===" + echo "Written to: $AMBIGUOUS_FILE" + jq -r '.[] | " #\(.issue_number): \(.broken_url) (\(.reason), \(.candidates | length) candidates)"' "$AMBIGUOUS_FILE" +fi + +# --- Step 8: Summary --- +echo "" +echo "=== Fixer Run $FIX_DATE Summary ===" +echo "Issues processed: $((PROCESSED + EXTERNAL_PROCESSED))" +echo "" +echo "Internal links:" +echo " Re-verified (now valid): $REVERIFIED_COUNT" +echo " Fixes found: $FIXED_COUNT" +echo " Unfixable: $UNFIXABLE_COUNT" +echo " Ambiguous (needs model): $AMBIGUOUS_COUNT" +echo "" +echo "External links:" +echo " Analyzed (commented): $EXTERNAL_ANALYZED" +echo " Re-verified (now valid): $EXTERNAL_REVERIFIED" +if [ "$DRY_RUN" = true ]; then + echo "" + echo "Mode: DRY RUN (no PRs created)" +fi +echo "Duration: ${SECONDS}s" diff --git a/skills/repoman/scripts/link-health-scanner.sh b/skills/repoman/scripts/link-health-scanner.sh new file mode 100755 index 0000000..9715d40 --- /dev/null +++ b/skills/repoman/scripts/link-health-scanner.sh @@ -0,0 +1,622 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Link Health Scanner +# Scans all repos for broken links, creates/closes GitHub issues, writes reports. +# +# Usage: +# bash link-health-scanner.sh # full run +# bash link-health-scanner.sh --dry-run # scan + report, no issues/PRs +# bash link-health-scanner.sh --issue-limit 3 # create at most 3 issues +# bash link-health-scanner.sh --dry-run --issue-limit 3 # combined +# ============================================================================= + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- CLI args --- +DRY_RUN=false +ISSUE_LIMIT=0 # 0 = unlimited + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) DRY_RUN=true; shift ;; + --issue-limit) ISSUE_LIMIT="$2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Resolve deployment constants (repos_dir, fork_owner) from ~/.repoman/config.json. +repoman_config + +# --- Configuration --- +REPORTS_DIR="${REPORTS_DIR:-$HOME/reports/link-health}" + +# Report-PR destination. The org main repo's docs/ folder feeds the docs site +# (rossoctl.dev) and cannot host machine-generated reports, so the standing +# report PR lands under automation-health/ in the automation repo. A single +# file, overwritten in place each run: trend tooling reconstructs history by +# replaying git commit parents, so we store state (not dated snapshots) and +# avoid the files-vs-diffs-on-Git anti-pattern (rossoctl/automation#44). +# TODO(RepoMan Phase 2): move report_target_repo to programs/link-health.json. +REPORT_TARGET_REPO="rossoctl/automation" +REPORT_TARGET_OWNER="${REPORT_TARGET_REPO%%/*}" +REPORT_TARGET_NAME="${REPORT_TARGET_REPO##*/}" +REPORT_TARGET_PATH="automation-health/link-health.md" + +# Provenance footer for the report-PR body. Resolved once at startup: the +# helper lazily writes _meta..json beside this skill and returns the pinned-SHA +# RepoMan attribution (blob/main fallback when the SHA cannot be resolved). +SKILL_FOOTER=$(skill_attribution "link-health-scanner") + +# Clone dir for the report target: honor an explicit MAIN_REPO_DIR override, +# else derive from the owner-namespaced layout ($REPOS_DIR//). +REPORT_TARGET_DIR="${MAIN_REPO_DIR:-$REPOS_DIR/$REPORT_TARGET_OWNER/$REPORT_TARGET_NAME}" + +# Fork remote name for the report-target push. Derived from the profile so it +# carries no org literal; a stale remote of this name is corrected below. +FORK_REMOTE="$FORK_OWNER-automation-fork" +SCAN_DATE=$(date -u +"%Y-%m-%d") +SCAN_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +MAX_HISTORY_ROWS=500 +ESCALATION_THRESHOLD=20 + +# --- Workspace setup --- +setup_workspace "link-scanner" +TMPDIR="$PROGRAM_TMPDIR" +mkdir -p "$REPORTS_DIR" + +# --- Scan ID --- +SCAN_ID=$(generate_scan_id "$REPORTS_DIR" "$SCAN_DATE") + +echo "=== Link Health Scan $SCAN_ID ===" +echo "Repos dir: $REPOS_DIR" +echo "Reports dir: $REPORTS_DIR" +if [ "$DRY_RUN" = true ]; then echo "Mode: DRY RUN (no issues, no PRs)"; fi +if [ "$ISSUE_LIMIT" -gt 0 ]; then echo "Issue limit: $ISSUE_LIMIT"; fi + +# --- Scan all repos --- +TOTAL_LINKS=0 +TOTAL_ERRORS=0 +REPOS_SCANNED=0 +REPOS_FAILED=0 + +# Collect all broken links into a single JSONL file +: > "$TMPDIR/broken.jsonl" + +# Drive the loop from the enrolled set (authoritative), statting each clone, +# rather than globbing "$REPOS_DIR"/*/ and filtering -- a "*/" glob silently +# drops a ".github" repo. enrolled_clone_dirs emits each enrolled owner/name +# whose clone exists; the here-string keeps the body in the current shell so +# counters accumulate. +ENROLLED=$(repoman_load_enrolled) || exit 1 +CLONES=$(enrolled_clone_dirs "$ENROLLED") + +while IFS= read -r full_repo; do + [ -n "$full_repo" ] || continue + repo_dir="$REPOS_DIR/$full_repo" + + echo "Scanning $full_repo..." + + # Key the temp file on the full owner/name (slash flattened to _), not the + # bare name: rossoctl/cortex and alice/cortex share a bare name, and keying + # on it would make the second scan overwrite the first (the [ ! -s ] guard + # cannot catch it -- the file is non-empty). That same-name-different-owner + # case is exactly what the owner-namespaced model exists to support. + safe_repo="${full_repo//\//_}" + LYCHEE_OUTPUT="$TMPDIR/lychee_${safe_repo}.json" + + # Run lychee -- scanner-level args applied to all repos + LYCHEE_SCANNER_ARGS=( + --format json + --scheme http --scheme https + --exclude 'localhost' --exclude '127\.0\.0\.1' --exclude 'localtest\.me' + --exclude 'example\.com' --exclude 'example\.org' + --exclude-all-private + --exclude-path 'node_modules' --exclude-path 'vendor' --exclude-path '\.claude' + --accept '200,204,206,403,429,502,503' + --exclude 'console\.cloud\.google\.com' + --timeout 10 + --max-retries 2 + --max-concurrency 8 + ) + + if [ -f "$repo_dir/.lychee.toml" ]; then + lychee "${LYCHEE_SCANNER_ARGS[@]}" --config "$repo_dir/.lychee.toml" "$repo_dir" > "$LYCHEE_OUTPUT" 2>/dev/null || true + else + lychee "${LYCHEE_SCANNER_ARGS[@]}" "$repo_dir" > "$LYCHEE_OUTPUT" 2>/dev/null || true + fi + + if [ ! -s "$LYCHEE_OUTPUT" ]; then + echo " WARN: lychee produced no output for $full_repo" + REPOS_FAILED=$((REPOS_FAILED + 1)) + continue + fi + + # Parse results + repo_total=$(jq '.total // 0' "$LYCHEE_OUTPUT") + repo_errors=$(jq '.errors // 0' "$LYCHEE_OUTPUT") + TOTAL_LINKS=$((TOTAL_LINKS + repo_total)) + TOTAL_ERRORS=$((TOTAL_ERRORS + repo_errors)) + REPOS_SCANNED=$((REPOS_SCANNED + 1)) + + # Extract broken links from the lychee report. The parsing/suppression/status + # normalization logic lives in extract-broken-links.sh so it can be unit-tested + # (see tests/test-extract-broken-links.sh). + "$SCRIPT_DIR/extract-broken-links.sh" \ + "$LYCHEE_OUTPUT" "$full_repo" "$repo_dir" \ + >> "$TMPDIR/broken.jsonl" 2>/dev/null || true + + echo " Links: $repo_total, Errors: $repo_errors" +done <<< "$CLONES" + +echo "" +echo "=== Scan complete ===" +echo "Repos scanned: $REPOS_SCANNED (failed: $REPOS_FAILED)" +echo "Total links: $TOTAL_LINKS, Total errors: $TOTAL_ERRORS" + +# --- Load previous scan for diffing --- +# Subset-run carry-forward (see docs/specs run-path, "Subset-aware stateful +# scanners"): when REPOMAN_SUBSET_REPOS is set, this run scanned only those +# repos. The diff must compare like-for-like, so the previous report is split +# into the findings for SCANNED repos (which participate in the diff) and the +# findings for UNSCANNED repos (carried forward verbatim into latest.json, +# never diffed, so their issues are never mis-closed as "fixed"). +ALL_PREV_BROKEN="$TMPDIR/all_prev_broken.jsonl" +PREV_BROKEN="$TMPDIR/prev_broken.jsonl" +CARRY_BROKEN="$TMPDIR/carry_broken.jsonl" +: > "$CARRY_BROKEN" +if [ -f "$REPORTS_DIR/latest.json" ]; then + jq -c '.broken[]?' "$REPORTS_DIR/latest.json" > "$ALL_PREV_BROKEN" 2>/dev/null || true +else + : > "$ALL_PREV_BROKEN" +fi +if [ -n "${REPOMAN_SUBSET_REPOS:-}" ]; then + # Build a JSON array of the scanned owner/name slugs once, pass via --argjson. + SCANNED_JSON=$(printf '%s\n' "$REPOMAN_SUBSET_REPOS" \ + | jq -R 'select(length>0)' | jq -s '.' 2>/dev/null || echo '[]') + # prev findings whose .repo IS in the scanned set -> participate in the diff + jq -c --argjson scanned "$SCANNED_JSON" \ + 'select(.repo as $r | $scanned | index($r))' \ + "$ALL_PREV_BROKEN" > "$PREV_BROKEN" 2>/dev/null || : > "$PREV_BROKEN" + # prev findings whose .repo is NOT in the scanned set -> carried forward + jq -c --argjson scanned "$SCANNED_JSON" \ + 'select(.repo as $r | ($scanned | index($r)) | not)' \ + "$ALL_PREV_BROKEN" > "$CARRY_BROKEN" 2>/dev/null || : > "$CARRY_BROKEN" +else + cp "$ALL_PREV_BROKEN" "$PREV_BROKEN" +fi + +# --- Compute diff using shared library --- +DIFF_KEY='[.repo, .file, .url] | join("|")' +read -r NEW_LINKS FIXED_LINKS RECURRING_LINKS < <( + diff_against_previous "$TMPDIR/broken.jsonl" "$PREV_BROKEN" "$DIFF_KEY" +) + +echo "Delta: +$NEW_LINKS new, -$FIXED_LINKS fixed, $RECURRING_LINKS recurring" + +# --- Count by category --- +BROKEN_INTERNAL=$(jq -s '[.[] | select(.category == "internal")] | length' "$TMPDIR/broken.jsonl" 2>/dev/null || echo 0) +BROKEN_EXTERNAL=$(jq -s '[.[] | select(.category == "external")] | length' "$TMPDIR/broken.jsonl" 2>/dev/null || echo 0) + +# --- Create GitHub issues for NEW broken links --- +ISSUES_CREATED=0 + +# new_keys.txt was already written by diff_against_previous + +while IFS='|' read -r issue_repo issue_file issue_url; do + [ -z "$issue_repo" ] && continue + + # Check issue limit + if [ "$ISSUE_LIMIT" -gt 0 ] && [ "$ISSUES_CREATED" -ge "$ISSUE_LIMIT" ]; then + echo " SKIP (issue limit $ISSUE_LIMIT reached): $issue_file:$issue_url" + continue + fi + + # Get the full broken link record + link_record=$(jq -c "select(.repo == \"$issue_repo\" and .file == \"$issue_file\" and .url == \"$issue_url\")" "$TMPDIR/broken.jsonl" | head -1) + [ -z "$link_record" ] && continue + + link_status=$(echo "$link_record" | jq -r '.status') + link_category=$(echo "$link_record" | jq -r '.category') + if [ "$link_category" = "internal" ]; then + category_label="broken-link/internal" + else + category_label="broken-link/external" + fi + + # Deduplication: skip if an open issue already exists for this link + existing=$(gh_issue_exists "$issue_repo" "Broken link in $issue_file: $issue_url" || true) + + if [ -n "$existing" ]; then + echo " Issue #$existing already exists for $issue_file:$issue_url" + continue + fi + + # Create issue + issue_title=":bug: Broken link in $issue_file: $issue_url" + # Truncate title if too long (GitHub limit is 256) + if [ ${#issue_title} -gt 250 ]; then + issue_title="${issue_title:0:247}..." + fi + + # Build verification note for ambiguous status codes + verify_note="" + case "$link_status" in + *403*) verify_note=" +> **Note:** This URL returned 403 (Forbidden). Some sites block automated scanners. The link may be valid when accessed from a browser. Please verify manually before fixing." ;; + *503*) verify_note=" +> **Note:** This URL returned 503 (Service Unavailable), which may indicate a temporarily unavailable service rather than a permanently broken link. Please verify manually before fixing." ;; + *429*) verify_note=" +> **Note:** This URL returned 429 (Too Many Requests). The link may be valid but rate-limited. Please verify manually before fixing." ;; + esac + + issue_body="## Describe the bug + +Broken link detected by automated link health scan. + +**Repo:** $issue_repo +**File:** $issue_file +**Broken URL:** $issue_url +**HTTP Status:** $link_status +**First detected:** $SCAN_DATE +**Scan ID:** $SCAN_ID +$verify_note +## Steps To Reproduce + +1. Open https://github.com/$issue_repo/blob/main/$issue_file +2. Click or follow the link to \`$issue_url\` +3. Observe $link_status error + +## Expected Behavior + +The link should resolve to valid documentation. + +## Additional Context + +Category: $link_category +Detected by RepoMan Link Health Scanner (cron: link-health-scanner)" + + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would create issue on $issue_repo: $issue_file:$issue_url ($link_category)" + ISSUES_CREATED=$((ISSUES_CREATED + 1)) + elif gh issue create --repo "$issue_repo" \ + --title "$issue_title" \ + --label "kind/bug,$category_label" \ + --body "$issue_body" 2>/dev/null; then + ISSUES_CREATED=$((ISSUES_CREATED + 1)) + echo " Created issue for $issue_file:$issue_url" + else + echo " WARN: Failed to create issue for $issue_file:$issue_url" + fi + + # Rate limit: small delay between issue creations + sleep 1 +done < "$TMPDIR/new_keys.txt" + +echo "Issues created: $ISSUES_CREATED" + +# --- Close issues for FIXED links --- +ISSUES_CLOSED=0 + +comm -13 "$TMPDIR/current_keys.txt" "$TMPDIR/prev_keys.txt" > "$TMPDIR/fixed_keys.txt" + +while IFS='|' read -r fix_repo fix_file fix_url; do + [ -z "$fix_repo" ] && continue + + # Get the category from the previous scan + link_record=$(jq -c "select(.repo == \"$fix_repo\" and .file == \"$fix_file\" and .url == \"$fix_url\")" "$PREV_BROKEN" | head -1) + link_category=$(echo "$link_record" | jq -r '.category // "internal"') + + if [ "$link_category" = "internal" ]; then + category_label="broken-link/internal" + else + category_label="broken-link/external" + fi + + # Find the open issue for this link (empty string if none exists) + issue_number=$(gh_issue_exists "$fix_repo" "Broken link in $fix_file: $fix_url" || true) + + # Skip if no matching issue was found + if [ -z "$issue_number" ]; then + continue + fi + + if [ "$DRY_RUN" = true ]; then + echo " [DRY RUN] Would close issue #$issue_number for $fix_file:$fix_url" + ISSUES_CLOSED=$((ISSUES_CLOSED + 1)) + elif close_issue_if_valid "$fix_repo" "$issue_number" \ + "Link verified as fixed in scan $SCAN_ID ($SCAN_DATE). Auto-closing."; then + ISSUES_CLOSED=$((ISSUES_CLOSED + 1)) + echo " Closed issue #$issue_number for $fix_file:$fix_url" + fi +done < "$TMPDIR/fixed_keys.txt" + +echo "Issues closed: $ISSUES_CLOSED" + +# --- Write latest.json --- +# Fresh findings for the scanned repos get the standard enrichment. On a subset +# run, the previous findings for UNSCANNED repos are carried forward verbatim +# (they already carry issue_number/first_detected from the prior report), so the +# baseline stays whole and the next full run diffs against a complete picture. +BROKEN_ARRAY=$(jq -s ' + [.[] | . + { + issue_number: null, + first_detected: "'"$SCAN_DATE"'", + context: "" + }] +' "$TMPDIR/broken.jsonl" 2>/dev/null || echo "[]") +if [ -s "$CARRY_BROKEN" ]; then + CARRY_ARRAY=$(jq -s '.' "$CARRY_BROKEN" 2>/dev/null || echo "[]") + BROKEN_ARRAY=$(jq -n --argjson a "$BROKEN_ARRAY" --argjson b "$CARRY_ARRAY" '$a + $b' 2>/dev/null || printf '%s' "$BROKEN_ARRAY") +fi + +# On a subset run the broken[] list is whole-set (carried forward), so keep the +# prior report's whole-set scalar totals rather than the subset-only counts, or +# the dashboard would read a subset repos_scanned/total_links_checked beside a +# whole-set broken[]. Per-run delta (new/fixed/recurring) still reflects this run. +if [ -n "${REPOMAN_SUBSET_REPOS:-}" ] && [ -f "$REPORTS_DIR/latest.json" ]; then + REPOS_SCANNED=$(jq -r '.repos_scanned // '"$REPOS_SCANNED" "$REPORTS_DIR/latest.json" 2>/dev/null || echo "$REPOS_SCANNED") + TOTAL_LINKS=$(jq -r '.total_links_checked // '"$TOTAL_LINKS" "$REPORTS_DIR/latest.json" 2>/dev/null || echo "$TOTAL_LINKS") +fi + +LATEST_JSON=$(cat << LATEST_EOF +{ + "scan_id": "$SCAN_ID", + "date": "$SCAN_TIME", + "duration_seconds": $SECONDS, + "model": "script", + "repos_scanned": $REPOS_SCANNED, + "repos_failed": $REPOS_FAILED, + "total_links_checked": $TOTAL_LINKS, + "broken": $BROKEN_ARRAY, + "delta": { + "new": $NEW_LINKS, + "fixed": $FIXED_LINKS, + "recurring": $RECURRING_LINKS + } +} +LATEST_EOF +) + +# A dry run must NOT write the baseline: overwriting latest.json would make the +# next real run see these findings as "recurring" (not "new") and skip filing +# them, and skip closing anything shown as fixed here. Preview only. +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would write latest.json (baseline left unchanged)" +else + write_report_latest "$REPORTS_DIR" "$LATEST_JSON" + echo "Wrote latest.json" +fi + +# --- Append to history.json --- +HISTORY_ROW=$(cat << HIST_EOF +{ + "scan_id": "$SCAN_ID", + "date": "$SCAN_TIME", + "repos_scanned": $REPOS_SCANNED, + "total_links_checked": $TOTAL_LINKS, + "broken_internal": $BROKEN_INTERNAL, + "broken_external": $BROKEN_EXTERNAL, + "new": $NEW_LINKS, + "fixed": $FIXED_LINKS, + "issues_created": $ISSUES_CREATED, + "issues_closed": $ISSUES_CLOSED +} +HIST_EOF +) + +# history.json is a full-set trend series (consumed by the dashboard). A subset +# run is a rare manual recovery over part of the enrolled set; appending its +# partial counts would distort the trend, so we skip the history append on a +# subset run. (Trade-off: that run's issues-created/closed counts are not +# recorded to history — accepted; see docs/specs run-path, "Subset-aware +# stateful scanners".) +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would append history.json row (skipped)" +elif [ -n "${REPOMAN_SUBSET_REPOS:-}" ]; then + echo "Subset run — skipping history.json append (trend series stays full-set)" +else + append_history_row "$REPORTS_DIR" "$HISTORY_ROW" "$MAX_HISTORY_ROWS" + echo "Appended to history.json" +fi + +# Note: _index.json registration is performed by the run-path orchestrator +# (scripts/repoman-run.sh), the sole index writer — not here. Running this +# scanner directly no longer updates the registry; `repoman run link-health` +# is the blessed path for an indexed run. (Run path: issue #107, epic #72.) + +# --- Update docs/link-health.md --- +# Build trend table from last 10 history entries (deduplicate same-day rows) +TREND_TABLE=$(jq -r ' + [group_by(.date | split("T")[0]) | .[] | last] | .[-10:] | reverse | .[] | + "| \(.date | split("T")[0] | split("-")[1:] | join("-")) | \(.broken_internal) | \(.broken_external) | \(if .new > .fixed then "+\(.new - .fixed)" elif .new < .fixed then "\(.new - .fixed)" else "0" end) |" +' "$REPORTS_DIR/history.json" 2>/dev/null || echo "| - | - | - | - |") + +# Build per-repo breakdown with issue counts +# Single org-wide query for all open scanner issues, then count client-side +issue_search=$(gh_with_backoff search issues "org:${REPORT_TARGET_REPO%%/*} in:title \"Broken link in\" state:open" --json repository --jq '.[].repository.nameWithOwner' 2>/dev/null || true) +# bash 3.2 (macOS default) has no associative arrays. Keep counts in a +# newline-delimited accumulator of "repocount" rows; repo keys are +# owner/name (no whitespace), so tab-splitting is unambiguous. +ISSUE_COUNTS="" # rows of "repocount" + +_lhs_ic_get() { # $1=repo -> prints its count, or 0 if absent + printf '%s\n' "$ISSUE_COUNTS" \ + | awk -F'\t' -v r="$1" '$1==r{c=$2; found=1} END{print (found ? c : 0)}' +} +_lhs_ic_incr() { # $1=repo -> increment its count by 1 + local cur rest + cur=$(_lhs_ic_get "$1") + # Drop any existing row for this repo. Guard the empty case so we do not + # feed awk a lone blank record (which would echo back an empty row). + if [ -n "$ISSUE_COUNTS" ]; then + rest=$(printf '%s\n' "$ISSUE_COUNTS" | awk -F'\t' -v r="$1" '$1!=r') + else + rest="" + fi + # Append the new count, keeping the accumulator free of leading blank rows. + if [ -n "$rest" ]; then + ISSUE_COUNTS="${rest} +$1 $(( cur + 1 ))" + else + ISSUE_COUNTS="$1 $(( cur + 1 ))" + fi +} + +if [ -n "$issue_search" ]; then + while IFS= read -r r; do + _lhs_ic_incr "$r" + done <<< "$issue_search" +fi + +# Single jq pass for per-repo internal/external counts +REPO_TABLE="" +repo_breakdown=$(jq -rs ' + group_by(.repo) | .[] | + { repo: .[0].repo, + internal: [.[] | select(.category == "internal")] | length, + external: [.[] | select(.category == "external")] | length } +' "$TMPDIR/broken.jsonl" 2>/dev/null || true) +if [ -n "$repo_breakdown" ]; then + while IFS= read -r row; do + repo=$(echo "$row" | jq -r '.repo') + short=$(echo "$repo" | cut -d/ -f2) + int_count=$(echo "$row" | jq -r '.internal') + ext_count=$(echo "$row" | jq -r '.external') + issue_count=$(_lhs_ic_get "$repo") + REPO_TABLE="${REPO_TABLE}| ${short} | ${int_count} | ${ext_count} | ${issue_count} | +" + done < <(echo "$repo_breakdown" | jq -c '.') +else + REPO_TABLE="| - | - | - | - |" +fi + +SCAN_TIME_ET=$(TZ="America/New_York" date +"%Y-%m-%d %H:%M ET") + +cat > "$TMPDIR/link-health.md" << DASHBOARD_EOF +# Link Health Report + +> Last scan: $SCAN_TIME_ET | Scan ID: $SCAN_ID + +## Summary + +| Metric | Value | +|--------|-------| +| Repos scanned | $REPOS_SCANNED | +| Total links checked | $TOTAL_LINKS | +| Broken (internal) | $BROKEN_INTERNAL | +| Broken (external) | $BROKEN_EXTERNAL | +| New since last scan | +$NEW_LINKS | +| Fixed since last scan | -$FIXED_LINKS | + +## Trend (last 10 scans) + +| Date | Internal | External | Delta | +|------|----------|----------|-------| +$TREND_TABLE + +## Broken Links by Repo + +| Repo | Internal | External | Issues | +|------|----------|----------|--------| +$REPO_TABLE + +*Issues counts open GitHub issues filed by the scanner; a broken link may not yet have an issue (due to per-run limits) or may share an issue with another link in the same file.* + +--- +*Generated by RepoMan Link Health Scanner. Do not edit manually.* +DASHBOARD_EOF + +# Commit and push dashboard +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would push $REPORT_TARGET_PATH to fork and create/update cross-fork PR against $REPORT_TARGET_REPO" + echo "[DRY RUN] Dashboard preview:" + cat "$TMPDIR/link-health.md" +else + if [ ! -d "$REPORT_TARGET_DIR/.git" ]; then + echo "ERROR: $REPORT_TARGET_DIR does not appear to be a git repository." + echo "Export MAIN_REPO_DIR or set REPOS_DIR so $REPORT_TARGET_REPO can be found:" + echo " export MAIN_REPO_DIR=$REPOS_DIR/$REPORT_TARGET_OWNER/$REPORT_TARGET_NAME" + exit 1 + fi + + cd "$REPORT_TARGET_DIR" + + # Ensure the fork remote exists AND points at the current target. set-url + # corrects a stale remote (e.g. one left by a prior deployment pointing at the + # old report repo); the || add branch handles the not-yet-registered case. + fork_url="https://github.com/$FORK_OWNER/${REPORT_TARGET_NAME}.git" + git remote set-url "$FORK_REMOTE" "$fork_url" 2>/dev/null \ + || git remote add "$FORK_REMOTE" "$fork_url" + + # Fetch fork's branch if it exists, otherwise create from main + if git fetch "$FORK_REMOTE" link-health/reports 2>/dev/null; then + git checkout -B link-health/reports "$FORK_REMOTE/link-health/reports" + else + git fetch "$FORK_REMOTE" main 2>/dev/null || true + git checkout -B link-health/reports "$FORK_REMOTE/main" 2>/dev/null \ + || git checkout -B link-health/reports + fi + + mkdir -p "$(dirname "$REPORT_TARGET_PATH")" + cp "$TMPDIR/link-health.md" "$REPORT_TARGET_PATH" + git add "$REPORT_TARGET_PATH" + git commit -s -m "docs: Update link health dashboard ($SCAN_ID)" 2>/dev/null || echo "No changes to commit" + git push "$FORK_REMOTE" link-health/reports 2>/dev/null || echo "WARN: Failed to push dashboard to fork" + + # Create or update standing cross-fork PR + existing_pr=$(gh api "repos/$REPORT_TARGET_REPO/pulls?head=$FORK_OWNER:link-health/reports&state=open" \ + --jq '.[0].number' 2>/dev/null || echo "") + + pr_body="## Summary + +Auto-updated by RepoMan Link Health Scanner. This PR is continuously updated with each scan. Merge when convenient. + +| Metric | Value | +|--------|-------| +| Repos scanned | $REPOS_SCANNED | +| Broken (internal) | $BROKEN_INTERNAL | +| Broken (external) | $BROKEN_EXTERNAL | +| New since last scan | +$NEW_LINKS | +| Fixed since last scan | -$FIXED_LINKS | + +## Related issue(s) + +- $REPORT_TARGET_REPO#1178 + +## Automation program + +$SKILL_FOOTER" + + if [ -z "$existing_pr" ] || [ "$existing_pr" = "null" ]; then + gh pr create --repo "$REPORT_TARGET_REPO" \ + --head "$FORK_OWNER:link-health/reports" --base main \ + --title "docs: Link health report (auto-updated)" \ + --body "$pr_body" 2>/dev/null || echo "WARN: Failed to create dashboard PR" + else + gh pr edit "$existing_pr" --repo "$REPORT_TARGET_REPO" --body "$pr_body" 2>/dev/null || true + fi +fi + +echo "Dashboard updated" + +# --- Escalation check --- +if [ "$NEW_LINKS" -gt "$ESCALATION_THRESHOLD" ]; then + echo "" + echo "ALERT: Link health scan found $NEW_LINKS new broken links (threshold: $ESCALATION_THRESHOLD)." + echo "This may indicate a bulk documentation change or a widespread external service outage." + echo "Review issues at https://github.com/$REPORT_TARGET_REPO/issues?q=label:broken-link" +fi + +# --- Summary --- +echo "" +echo "=== Scan $SCAN_ID Summary ===" +echo "Repos: $REPOS_SCANNED scanned, $REPOS_FAILED failed" +echo "Links: $TOTAL_LINKS checked, $((BROKEN_INTERNAL + BROKEN_EXTERNAL)) broken ($BROKEN_INTERNAL internal, $BROKEN_EXTERNAL external)" +echo "Delta: +$NEW_LINKS new, -$FIXED_LINKS fixed, $RECURRING_LINKS recurring" +echo "Issues: $ISSUES_CREATED created, $ISSUES_CLOSED closed" +echo "Duration: ${SECONDS}s" diff --git a/skills/repoman/scripts/parse-diff-map.sh b/skills/repoman/scripts/parse-diff-map.sh new file mode 100755 index 0000000..61e4006 --- /dev/null +++ b/skills/repoman/scripts/parse-diff-map.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +set -euo pipefail + +# parse-diff-map.sh — Convert a unified diff into a structured JSON diff_map. +# +# Each file in the diff becomes an object with a path and array of hunks. +# Each hunk contains an array of lines with their absolute line numbers, +# the side they appear on (LEFT for removed, RIGHT for added/context), +# and their type (added, removed, context). +# +# This is used by the PR review fixer to give the agent precise line numbers +# for inline review comments, avoiding the need for hunk arithmetic in the LLM. +# +# Usage: +# parse-diff-map.sh +# cat some.diff | parse-diff-map.sh - +# +# Output (stdout): JSON array +# [ +# { +# "path": "src/foo.ts", +# "hunks": [ +# { +# "lines": [ +# {"line": 42, "side": "RIGHT", "type": "added", "content": "new code"}, +# {"line": 10, "side": "LEFT", "type": "removed", "content": "old code"}, +# {"line": 43, "side": "RIGHT", "type": "context", "content": "unchanged"} +# ] +# } +# ] +# } +# ] +# +# Exit codes: +# 0 - success (outputs JSON array, possibly empty []) +# 1 - usage error (no file argument or file not found) + +# --- Argument handling --- +if [ $# -lt 1 ]; then + echo "Usage: parse-diff-map.sh " >&2 + echo " parse-diff-map.sh - (read from stdin)" >&2 + exit 1 +fi + +DIFF_INPUT="$1" + +if [ "$DIFF_INPUT" != "-" ] && [ ! -f "$DIFF_INPUT" ]; then + echo "ERROR: File not found: $DIFF_INPUT" >&2 + exit 1 +fi + +# Empty input produces empty array +if [ "$DIFF_INPUT" != "-" ] && [ ! -s "$DIFF_INPUT" ]; then + echo "[]" + exit 0 +fi + +# --- AWK parser --- +# The parser is a state machine that tracks: +# - Which file we are in (from +++ b/... lines) +# - Which hunk we are in (from @@ lines) +# - Current line numbers for old (left) and new (right) sides +# +# JSON is built incrementally with printf to avoid buffering issues. + +awk ' +BEGIN { + printf "[" + first_file = 1 + in_hunk = 0 + started_file = 0 +} + +# New file boundary: "diff --git a/... b/..." +/^diff --git/ { + if (in_hunk) { + printf "]}" + in_hunk = 0 + } + if (started_file) { + printf "]}" + } + started_file = 0 + first_hunk = 1 + src_path = "" + next +} + +# Source path from --- line (used as fallback for deleted files) +/^--- a\// { + src_path = substr($0, 7) + next +} + +# Skip other --- lines (e.g. "--- /dev/null" for new files) +/^--- / { next } + +# File path from +++ line (destination side) +/^\+\+\+ b\// { + path = substr($0, 7) + if (!first_file) printf "," + printf "{\"path\":\"%s\",\"hunks\":[", path + first_file = 0 + started_file = 1 + next +} + +# Deleted file: +++ /dev/null — use the source path +/^\+\+\+ \/dev\/null/ { + if (src_path != "") { + if (!first_file) printf "," + printf "{\"path\":\"%s\",\"hunks\":[", src_path + first_file = 0 + started_file = 1 + } + next +} + +# Hunk header: @@ -old_start[,old_count] +new_start[,new_count] @@ +/^@@ / { + if (in_hunk) { + # Close the previous hunk + printf "]}" + } + if (!first_hunk) printf "," + first_hunk = 0 + in_hunk = 1 + first_line = 1 + + # Parse the line numbers from the hunk header. + # Format: @@ -[,] +[,] @@ + # We strip the leading "@@ -" then split on space to get the two ranges. + header = $0 + sub(/^@@ -/, "", header) + split(header, parts, " ") + + # parts[1] = "old_start,old_count" (or just "old_start") + split(parts[1], old_parts, ",") + old_line = old_parts[1] + 0 + + # parts[2] = "+new_start,new_count" (or just "+new_start") + new_range = parts[2] + sub(/^\+/, "", new_range) + split(new_range, new_parts, ",") + new_line = new_parts[1] + 0 + + printf "{\"lines\":[" + next +} + +# Added line (starts with +, not part of header) +in_hunk && /^\+/ { + content = substr($0, 2) + gsub(/\\/, "\\\\", content) + gsub(/"/, "\\\"", content) + gsub(/\t/, "\\t", content) + gsub(/\r/, "", content) + gsub(/\f/, "\\f", content) + gsub(/\b/, "\\b", content) + if (!first_line) printf "," + printf "{\"line\":%d,\"side\":\"RIGHT\",\"type\":\"added\",\"content\":\"%s\"}", new_line, content + new_line++ + first_line = 0 + next +} + +# Removed line (starts with -) +in_hunk && /^-/ { + content = substr($0, 2) + gsub(/\\/, "\\\\", content) + gsub(/"/, "\\\"", content) + gsub(/\t/, "\\t", content) + gsub(/\r/, "", content) + gsub(/\f/, "\\f", content) + gsub(/\b/, "\\b", content) + if (!first_line) printf "," + printf "{\"line\":%d,\"side\":\"LEFT\",\"type\":\"removed\",\"content\":\"%s\"}", old_line, content + old_line++ + first_line = 0 + next +} + +# Context line (starts with space) +in_hunk && /^ / { + content = substr($0, 2) + gsub(/\\/, "\\\\", content) + gsub(/"/, "\\\"", content) + gsub(/\t/, "\\t", content) + gsub(/\r/, "", content) + gsub(/\f/, "\\f", content) + gsub(/\b/, "\\b", content) + if (!first_line) printf "," + printf "{\"line\":%d,\"side\":\"RIGHT\",\"type\":\"context\",\"content\":\"%s\"}", new_line, content + old_line++ + new_line++ + first_line = 0 + next +} + +END { + if (in_hunk) printf "]}" + if (started_file) printf "]}" + printf "]\n" +} +' "$DIFF_INPUT" diff --git a/skills/repoman/scripts/pr-review-fixer.sh b/skills/repoman/scripts/pr-review-fixer.sh new file mode 100755 index 0000000..3568623 --- /dev/null +++ b/skills/repoman/scripts/pr-review-fixer.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +set -euo pipefail + +# PR Review Fixer — State Transition Manager (scanner/fixer pattern) +# Manages the in_progress/reviewed state for the PR review bot. +# Called by the agent at the start (begin) and end (finalize) of a review cycle. + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pr-review-lib.sh" + +# --- Configuration --- +BOT_USER="clawgenti" +REVIEW_MARKER="' \ + | tail -1 \ + | sed -E 's//\1/' \ + || true) + + if [ -n "$reviewed_sha" ] && [ "$reviewed_sha" = "$head_sha" ]; then + [ "$VERBOSE" = true ] && echo " Confirmed: reviewed at $head_sha" >&2 + + # Increment reviewed counter + count=$(cat "$TMPDIR/reviewed_count.txt") + echo "$((count + 1))" > "$TMPDIR/reviewed_count.txt" + + # Move from in_progress to reviewed + if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would move $repo#$number to reviewed" >&2 + else + jq --arg repo "$repo" --argjson num "$number" --arg sha "$head_sha" --arg ts "$NOW" \ + '.in_progress = [.in_progress[] | select(.repo != $repo or .number != $num)] | + .reviewed = [.reviewed[] | select(.repo != $repo or .number != $num)] + [{"repo": $repo, "number": $num, "head_sha": $sha, "reviewed_at": $ts}]' \ + "$TMPDIR/state.json" > "$TMPDIR/state_tmp.json" + mv "$TMPDIR/state_tmp.json" "$TMPDIR/state.json" + fi + else + [ "$VERBOSE" = true ] && echo " Not yet reviewed (will retry next cycle)" >&2 + + # Increment failed counter + count=$(cat "$TMPDIR/failed_count.txt") + echo "$((count + 1))" > "$TMPDIR/failed_count.txt" + fi + done < "$TMPDIR/in_progress_entries.jsonl" + + # Read final counters + PRS_REVIEWED=$(cat "$TMPDIR/reviewed_count.txt") + PRS_FAILED=$(cat "$TMPDIR/failed_count.txt") + + # Write updated state and history + if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would write state.json and fixer-history.json to $REPORTS_DIR" >&2 + else + pr_review_state_write "$TMPDIR/state.json" "$REPORTS_DIR/state.json" "pr-review-fixer" + + FIXER_ROW=$(jq -n \ + --arg date "$NOW" \ + --argjson prs_processed "$COUNT" \ + --argjson prs_reviewed "$PRS_REVIEWED" \ + --argjson prs_failed "$PRS_FAILED" \ + '{date: $date, prs_processed: $prs_processed, prs_reviewed: $prs_reviewed, prs_failed: $prs_failed}') + + append_history_row "$REPORTS_DIR" "$FIXER_ROW" "$MAX_HISTORY_ROWS" "fixer-history.json" + [ "$VERBOSE" = true ] && echo "Appended to $REPORTS_DIR/fixer-history.json" >&2 + fi + + # Output summary + jq -n \ + --argjson prs_processed "$COUNT" \ + --argjson prs_reviewed "$PRS_REVIEWED" \ + --argjson prs_failed "$PRS_FAILED" \ + '{prs_processed: $prs_processed, prs_reviewed: $prs_reviewed, prs_failed: $prs_failed}' + exit 0 +fi diff --git a/skills/repoman/scripts/pr-review-impact.sh b/skills/repoman/scripts/pr-review-impact.sh new file mode 100755 index 0000000..e54c2ac --- /dev/null +++ b/skills/repoman/scripts/pr-review-impact.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +# PR Review Impact — computes clawgenti review impact on time-to-merge. +# +# A clawgenti review is a REVIEW SUBMISSION (GET /pulls/N/reviews), not the PR +# description. Its body carries the marker "" and a real +# submitted_at. clawgenti ALSO comments on Dependabot PRs as the dep-bump fixer, +# so detection uses a two-layer filter: (1) a reviewed-by:clawgenti search +# prefilter (drops comment-only/dep-bump PRs at the search stage), and (2) a +# required marked review on each candidate. +# +# Per repo, activation = earliest marked-review submitted_at. Merged PRs are +# segmented reviewed/unreviewed and before/after that repo's own activation +# (split on createdAt). Buckets aggregate across repos. Read-only against +# GitHub; writes only to the reports dir. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +# --- Configuration --- +BOT_USER="clawgenti" +REVIEW_MARKER="' \ + | tail -1 \ + | sed -E 's//\1/' \ + || true) + + if [ -n "$last_reviewed_sha" ] && [ "$last_reviewed_sha" = "$head_sha" ]; then + [ "$VERBOSE" = true ] && echo " Skipping: already reviewed at $head_sha (API check)" >&2 + # Backfill state so we skip faster next time + jq --arg repo "$repo" --argjson num "$number" --arg sha "$head_sha" --arg ts "$SCAN_TIME" \ + '.reviewed += [{"repo": $repo, "number": $num, "head_sha": $sha, "reviewed_at": $ts}]' \ + "$TMPDIR/state.json" > "$TMPDIR/state_tmp.json" + mv "$TMPDIR/state_tmp.json" "$TMPDIR/state.json" + SKIPPED_REVIEWED=$((SKIPPED_REVIEWED + 1)) + continue + fi + + [ "$VERBOSE" = true ] && echo " Eligible for review" >&2 + jq --argjson pr "$candidate" '. += [$pr]' "$QUEUE_FILE" > "$TMPDIR/queue_tmp.json" + mv "$TMPDIR/queue_tmp.json" "$QUEUE_FILE" + + done < "$CANDIDATES_FILE" +fi + +ELIGIBLE=$(jq 'length' "$QUEUE_FILE") + +# --- Step 4: Write reports (skipped in dry-run mode) --- + +if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] Would write latest.json, state.json, history.json to $REPORTS_DIR" >&2 + echo "[DRY RUN] Summary: repos=$REPOS_SCANNED candidates=$CANDIDATES_FOUND eligible=$ELIGIBLE skipped_reviewed=$SKIPPED_REVIEWED skipped_in_progress=$SKIPPED_IN_PROGRESS evicted=$EVICTED" >&2 +else + # latest.json + LATEST_JSON=$(jq -n \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson repos_scanned "$REPOS_SCANNED" \ + --argjson candidates_found "$CANDIDATES_FOUND" \ + --argjson eligible_prs "$(cat "$QUEUE_FILE")" \ + '{scan_id: $scan_id, date: $date, repos_scanned: $repos_scanned, candidates_found: $candidates_found, eligible_prs: $eligible_prs}') + + # Subset carry-forward (steps 3/4 of the run-path spec's contract): on a + # narrowed run, merge the prior latest.json's eligible_prs for UNSCANNED repos + # back in, so the fixer's queue (which reads .eligible_prs) is not truncated to + # just the scanned repos. + # + # BUT the carried set is a stale snapshot: a PR the fixer has since reviewed + # (now in state.reviewed / in_progress, PR still open) would be re-listed as + # eligible, and `pr-review-fixer.sh begin` re-queues every eligible entry + # unconditionally -> a duplicate review of an unchanged PR. So filter the + # carried set against the CURRENT state ($TMPDIR/state.json, already loaded and + # evicted this run): drop any carried entry whose repo/number is in + # reviewed/in_progress at the SAME head_sha (mirrors the per-candidate skip at + # the review step). A head_sha change means the PR moved on and is legitimately + # eligible again, so it is kept. + if [ -n "${REPOMAN_SUBSET_REPOS:-}" ] && [ -f "$REPORTS_DIR/latest.json" ]; then + SCANNED_JSON=$(printf '%s\n' "$REPOMAN_SUBSET_REPOS" \ + | jq -R 'select(length>0)' | jq -s '.' 2>/dev/null || echo '[]') + # "handled" = {repo, number, head_sha} already reviewed/in_progress now. + HANDLED_JSON=$(jq -c '[(.reviewed[]?, .in_progress[]?) | {repo, number, head_sha}]' \ + "$TMPDIR/state.json" 2>/dev/null || echo '[]') + CARRY_ELIGIBLE=$(jq -c --argjson scanned "$SCANNED_JSON" --argjson handled "$HANDLED_JSON" ' + [ .eligible_prs[]? + | select(.repo as $r | ($scanned | index($r)) | not) # unscanned only + | select( . as $e + | ($handled | any(.repo == $e.repo and .number == $e.number + and .head_sha == $e.head_sha)) | not ) ] # not already handled at this SHA + ' "$REPORTS_DIR/latest.json" 2>/dev/null || echo '[]') + LATEST_JSON=$(printf '%s' "$LATEST_JSON" \ + | jq -c --argjson carry "$CARRY_ELIGIBLE" '.eligible_prs += $carry' 2>/dev/null || printf '%s' "$LATEST_JSON") + fi + + write_report_latest "$REPORTS_DIR" "$LATEST_JSON" + [ "$VERBOSE" = true ] && echo "Wrote $REPORTS_DIR/latest.json" >&2 + + # state.json (envelope v1, atomic write from file) + pr_review_state_write "$TMPDIR/state.json" "$REPORTS_DIR/state.json" "pr-review-scanner" + [ "$VERBOSE" = true ] && echo "Wrote $REPORTS_DIR/state.json" >&2 + + # history.json — skip on a subset run (full-set trend series; see run-path spec). + if [ -n "${REPOMAN_SUBSET_REPOS:-}" ]; then + [ "$VERBOSE" = true ] && echo "Subset run — skipping history.json append" >&2 + else + HISTORY_ROW=$(jq -n \ + --arg scan_id "$SCAN_ID" \ + --arg date "$SCAN_TIME" \ + --argjson repos_scanned "$REPOS_SCANNED" \ + --argjson eligible "$ELIGIBLE" \ + --argjson skipped_in_progress "$SKIPPED_IN_PROGRESS" \ + --argjson skipped_reviewed "$SKIPPED_REVIEWED" \ + --argjson evicted "$EVICTED" \ + '{scan_id: $scan_id, date: $date, repos_scanned: $repos_scanned, eligible: $eligible, skipped_in_progress: $skipped_in_progress, skipped_reviewed: $skipped_reviewed, evicted: $evicted}') + + append_history_row "$REPORTS_DIR" "$HISTORY_ROW" "$MAX_HISTORY_ROWS" + [ "$VERBOSE" = true ] && echo "Appended to $REPORTS_DIR/history.json" >&2 + fi +fi + +# --- Output (backward compatible) --- +cat "$QUEUE_FILE" diff --git a/skills/repoman/scripts/program-lib.sh b/skills/repoman/scripts/program-lib.sh new file mode 100644 index 0000000..566d169 --- /dev/null +++ b/skills/repoman/scripts/program-lib.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Aggregator for the automation library modules. Kept as the stable entrypoint +# so program scripts can `source "$SCRIPT_DIR/program-lib.sh"` unchanged. +# +# Source this file at the top of program scripts: +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/program-lib.sh" +# +# Prerequisites: jq, gh (GitHub CLI), coreutils (comm, sort, wc, mktemp) +_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +. "$_LIB_DIR/core.sh" +# shellcheck source=/dev/null +. "$_LIB_DIR/github-api.sh" +# shellcheck source=/dev/null +. "$_LIB_DIR/fork.sh" +# shellcheck source=/dev/null +. "$_LIB_DIR/org.sh" +# shellcheck source=/dev/null +. "$_LIB_DIR/repoman-requirements.sh" diff --git a/skills/repoman/scripts/repo-sync.sh b/skills/repoman/scripts/repo-sync.sh new file mode 100644 index 0000000..e99ef36 --- /dev/null +++ b/skills/repoman/scripts/repo-sync.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# repo-sync: keep the enrolled repo clones current. Reads the enrolled set from +# ~/.repoman/repos.json, clones-if-missing / pulls-if-present each repo under +# $REPOS_DIR///. Read-only toward GitHub. Per-repo failures are +# logged and non-fatal; every other failure is surfaced to the caller. +set -uo pipefail + +# --- Load shared library --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/program-lib.sh" + +usage() { + cat <<'EOF' +Usage: repo-sync.sh [--refresh-skills] [--dry-run] [--help] + +Sync enrolled repo clones (from ~/.repoman/repos.json) under +///. Clones if missing, pulls (--ff-only) if present. +Read-only toward GitHub. A per-repo clone/pull failure is logged and does not +abort the run. + + --refresh-skills Also pull a local agent-skills clone. Setting the + REPOMAN_SKILLS_DIR env var alone also triggers this; + the var selects the clone path (default ~/agent-skills). + --dry-run Preview only: print whether each enrolled repo would be + cloned (missing) or pulled (present), touch nothing on + disk or GitHub, and exit 0. Side-effect-free. + --help Show this help and exit 0. +EOF +} + +REFRESH_SKILLS=0 +DRY_RUN=false +while [ $# -gt 0 ]; do + case "$1" in + --refresh-skills) REFRESH_SKILLS=1; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --help) usage; exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +# Fail-loud inputs (rc surfaced): config sets REPOS_DIR/FORK_OWNER; enrolled set. +# repo-sync only uses REPOS_DIR, but repoman_config also validates and exports +# FORK_OWNER as a required key, so this run inherits a dependency on a key it +# does not itself use. Tracked for v0.1.1 (rossoctl/automation#98): let callers +# validate only the config keys they need. +repoman_config || exit 1 +ENROLLED=$(repoman_get_repos) || exit 1 + +pulled=0; cloned=0; failed=0; enrolled=0 +failures="" + +# valid_slug SLUG — true if SLUG is a safe single path segment for an owner or +# repo name. repos.json is external input; an entry like "../../etc" or an +# absolute path would let a clone/mkdir escape the repos_dir tree. Restrict to +# GitHub-legal characters ([A-Za-z0-9._-]) and reject empty or "." / ".." +# (a "." component would still resolve inside the tree but is never a real repo, +# and ".." escapes it). Slashes cannot appear: they are already consumed by the +# owner/name split, so a "name" of "../../escape" arrives here as "../../escape" +# and is rejected for the ".." components and the "/" it still contains. +valid_slug() { + local s="$1" + case "$s" in + ""|"."|"..") return 1 ;; + *[!A-Za-z0-9._-]*) return 1 ;; # any char outside the safe set (incl. "/") + esac + return 0 +} + +# Scratch file for capturing a per-repo git command's stderr, so its one-line +# error can be attached to that repo's summary entry (fail-loud: check mktemp's rc). +giterr=$(mktemp) || { echo "ERROR: mktemp failed" >&2; exit 1; } +trap 'rm -f "$giterr"' EXIT + +# git_error_line FILE — print the one line of git's stderr that names the cause. +# git writes progress first ("Cloning into '...'", "From https://...") and the +# actual "fatal:"/"error:"/"remote:" line last, so head -n1 would grab the +# useless progress line. Prefer the last fatal/error/remote line; fall back to +# the last non-empty line; finally the first line. Never fails (display only). +git_error_line() { + local f="$1" line="" + line=$(grep -iE '^(fatal|error|remote):' "$f" 2>/dev/null | tail -n1) + if [ -z "$line" ]; then + line=$(grep -v '^[[:space:]]*$' "$f" 2>/dev/null | tail -n1) + fi + if [ -z "$line" ]; then + line=$(head -n1 "$f" 2>/dev/null) + fi + printf '%s' "$line" +} + +while IFS= read -r full; do + [ -n "$full" ] || continue + enrolled=$((enrolled + 1)) + owner="${full%%/*}"; name="${full#*/}" + # Reject path-traversal / unsafe entries before any mkdir/clone touches the + # filesystem. Counted failed and named, like any other per-repo failure. + if ! valid_slug "$owner" || ! valid_slug "$name"; then + failed=$((failed + 1)); failures="$failures $full (invalid owner/name)"$'\n' + continue + fi + dir="$REPOS_DIR/$owner/$name" + if [ -d "$dir/.git" ]; then + if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] would pull $full (present at $dir)"; pulled=$((pulled + 1)); continue + fi + if git -C "$dir" pull --ff-only 2>"$giterr"; then + pulled=$((pulled + 1)) + else + msg=$(git_error_line "$giterr") + failed=$((failed + 1)); failures="$failures $full (pull): $msg"$'\n' + fi + else + if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] would clone $full -> $dir"; cloned=$((cloned + 1)); continue + fi + if ! mkdir -p "$REPOS_DIR/$owner"; then + failed=$((failed + 1)); failures="$failures $full (mkdir failed)"$'\n' + continue + fi + if git clone "https://github.com/$owner/$name.git" "$dir" 2>"$giterr"; then + cloned=$((cloned + 1)) + else + msg=$(git_error_line "$giterr") + failed=$((failed + 1)); failures="$failures $full (clone): $msg"$'\n' + fi + fi +done <<< "$ENROLLED" + +echo "repo-sync: pulled $pulled cloned $cloned failed $failed" +if [ "$failed" -gt 0 ]; then + echo "failed repos:" + printf '%s' "$failures" +fi + +# Exit model: a per-repo failure is non-fatal so a single bad repo does not +# abort the sync of the rest. But if EVERY enrolled repo failed, the run +# accomplished nothing and must not report success -- that signals a systemic +# fault (bad config, network down, all clones broken), not one flaky repo. +rc=0 +if [ "$enrolled" -gt 0 ] && [ "$failed" -eq "$enrolled" ]; then + echo "ERROR: all $enrolled enrolled repos failed to sync" >&2 + rc=1 +fi + +# --- optional skills refresh (opt-in; fatal on failure; no bootstrap-clone) --- +# Two triggers, per spec: the --refresh-skills flag, OR a set REPOMAN_SKILLS_DIR +# (the flag wins if both are given, but either alone opts in). Setting the env +# var alone must not be a silent no-op, so it triggers the refresh here too. +if [ "$REFRESH_SKILLS" -eq 1 ] || [ -n "${REPOMAN_SKILLS_DIR:-}" ]; then + skills_dir="${REPOMAN_SKILLS_DIR:-$HOME/agent-skills}" + if [ "$DRY_RUN" = true ]; then + echo "[DRY RUN] would refresh skills clone at $skills_dir" + elif [ -d "$skills_dir/.git" ]; then + if git -C "$skills_dir" pull --ff-only; then + echo "repo-sync: refreshed skills clone at $skills_dir" + else + echo "ERROR: skills refresh failed: git pull in $skills_dir" >&2 + rc=1 + fi + else + # Trigger-neutral wording: the refresh may have been requested by either the + # flag or the env var, so name neither. + echo "ERROR: skills refresh requested but no skills clone at $skills_dir" >&2 + echo " (repo-sync does not bootstrap the skills clone; create it first)" >&2 + rc=1 + fi +fi +exit "$rc" diff --git a/skills/repoman/scripts/repoman-check.sh b/skills/repoman/scripts/repoman-check.sh new file mode 100755 index 0000000..79aea1a --- /dev/null +++ b/skills/repoman/scripts/repoman-check.sh @@ -0,0 +1,387 @@ +#!/usr/bin/env bash +# RepoMan invocation-time capability checker. +# +# Verifies, BEFORE a program runs, that the active PAT holds the scopes the +# program's SKILL.md declared (programs/.json: pat_scopes) and reports +# which enrolled repos are missing the program's required labels +# (labels_required). Scope gaps are hard failures (exit non-zero); label gaps +# are warnings (exit 0 unless a scope gate also failed) because a missing +# label degrades a program's usefulness but does not make it unsafe to run. +# +# Usage: +# repoman-check.sh --program [--create-missing-labels] +# +# Read-only by default. --create-missing-labels is the only flag that mutates +# (creates missing labels), and only when the PAT's scopes intersect the +# accepted-scopes for the labels endpoint (i.e. the PAT actually CAN create +# labels there). +# +# ## Portability +# Targets bash 3.2+ (macOS default): no mapfile, no declare -A, no +# associative arrays. "Sets" are newline-accumulator strings. +# +# set -uo pipefail (NOT -e): this script aggregates gaps across every repo +# and every required label before deciding whether to exit non-zero, so one +# repo's gap must not abort the loop before later repos are checked. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +. "$SCRIPT_DIR/program-lib.sh" + +usage() { + cat <<'EOF' +Usage: repoman-check.sh --program [--create-missing-labels] + +Invocation-time capability check for a RepoMan program. Reads +programs/.json (pat_scopes, labels_required) and: + + 1. Scope check (hard gate): confirms the active PAT (`gh api -i user`) + holds every scope in pat_scopes. A fine-grained PAT (no X-OAuth-Scopes + header) always hard-fails this gate. Missing scopes hard-fail too. + + 2. Label check (per enrolled repo, warning only): for each label in + labels_required missing from a repo, reports one of: + - PAT can create + --create-missing-labels: creates it via + `gh label create` and reports success. + - PAT can create + no flag: prints the exact `gh label create` + command to run. + - PAT cannot create (its scopes do not cover the labels endpoint's + accepted scopes): points at a wider PAT or the GitHub web UI + (Settings -> Labels). Never prints a `gh label create` command in + this case, since running it would just fail. + +Flags: + --program Required. Selects programs/.json. + --create-missing-labels Create missing labels where the PAT can. + --help Show this help and exit 0. + +Exit status: non-zero iff any scope gate failed. Label gaps alone exit 0. + +Path overrides (same as the rest of RepoMan): + REPOMAN_PROGRAMS_DIR (default ~/.repoman/programs) + REPOMAN_REPOS_FILE (default ~/.repoman/repos.json) +EOF +} + +# ----------------------------------------------------------------------------- +# Aggregated findings (newline-accumulator strings; bash 3.2 has no arrays of +# structs). scope_gate_failed is the only thing that flips the exit code. +# ----------------------------------------------------------------------------- +scope_gate_failed=0 +findings="" + +add_finding() { + findings="${findings}$1"$'\n' +} + +# ----------------------------------------------------------------------------- +# check_scopes +# Hard gate. Sets PAT_SCOPES (newline-list of the token's own scopes) as a +# side effect for later label-capability checks. Returns via scope_gate_failed +# / findings, not via exit -- callers must keep going to check labels too. +# ----------------------------------------------------------------------------- +PAT_SCOPES="" + +check_scopes() { + local required="$1" + local hdr scopes_line scopes_value req missing="" + + # gh_with_backoff writes its error to stderr and returns 1 on a non-rate-limit + # failure, so an unguarded `$(...)` would capture an empty string and fall + # through to the empty-scopes branch below, misreporting a transport/auth + # failure as "fine-grained token". Check the status explicitly ($SCRIPT set + # -e is off by design). tr -d '\r' must run on its own line, not piped inside + # the substitution -- there the status would be tr's, not gh's. + if ! hdr=$(gh_with_backoff api -i user); then + add_finding "SCOPE FAIL: could not query the active token via 'gh api -i user' (see stderr above). This is a transport or authentication failure, not a token-type problem -- check 'gh auth status' and network reachability, then re-run." + scope_gate_failed=1 + PAT_SCOPES="" + return + fi + hdr=$(printf '%s' "$hdr" | tr -d '\r') + + scopes_line=$(printf '%s\n' "$hdr" | grep -i '^X-OAuth-Scopes:' | head -1) + scopes_value="${scopes_line#*:}" + # trim leading/trailing whitespace + scopes_value="${scopes_value#"${scopes_value%%[![:space:]]*}"}" + scopes_value="${scopes_value%"${scopes_value##*[![:space:]]}"}" + + if [ -z "$scopes_value" ]; then + add_finding "SCOPE FAIL: token appears to be a fine-grained personal access token (no X-OAuth-Scopes header). This checker validates classic-PAT OAuth scopes; a fine-grained token cannot be verified the same way. Re-run with a classic PAT that has: $(printf '%s' "$required" | tr '\n' ' ')" + scope_gate_failed=1 + PAT_SCOPES="" + return + fi + + # Normalize token scopes to a newline list for membership tests. + PAT_SCOPES=$(printf '%s' "$scopes_value" | tr ',' '\n' | while IFS= read -r s || [ -n "$s" ]; do + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + [ -n "$s" ] && printf '%s\n' "$s" + done) + + while IFS= read -r req || [ -n "$req" ]; do + [ -n "$req" ] || continue + if ! printf '%s\n' "$PAT_SCOPES" | grep -qxF "$req"; then + missing="${missing}${req}"$'\n' + fi + done < +# Prints the newline-list of accepted scopes for the labels endpoint, +# caching by the repo's visibility (private/public) so repeated calls for +# repos of the same visibility do not re-probe. +probe_accepted_scopes() { + local repo="$1" is_private hdr line value cache_file vis_file vis_key + + # Cache the visibility lookup per repo so `gh api repos/` fires at most + # once per repo (not once per missing label), then key the accepted-scope + # probe on the visibility class so it fires at most once per class. + vis_key=$(printf '%s' "$repo" | tr '/' '_') + vis_file="$CACHE_DIR/vis_$vis_key" + if [ -f "$vis_file" ]; then + is_private=$(cat "$vis_file") + else + # Guard the visibility lookup and let its stderr through (no 2>/dev/null): + # on failure the empty result would fall through the case to the public + # branch, silently misclassifying a private repo, and caching it would make + # that stick. Signal the failure to the caller (via a stdout sentinel, since + # this function runs in a $(...) subshell where add_finding would be lost) + # and do NOT cache it. + if is_private=$(gh_with_backoff api "repos/$repo" --jq '.private'); then + printf '%s' "$is_private" > "$vis_file" + else + printf '%s\n' "__VIS_FAILED__" + return + fi + fi + case "$is_private" in + true) cache_file="$CACHE_DIR/accepted_private" ;; + *) cache_file="$CACHE_DIR/accepted_public" ;; + esac + + if [ -f "$cache_file" ]; then + cat "$cache_file" + return + fi + + # Guard the probe: on failure gh_with_backoff returns 1 with empty stdout, and + # an unchecked capture would yield an empty accepted-scopes list -- read as + # "PAT cannot create labels" for a token that can. Emit a sentinel instead and + # do NOT cache it: caching a transient failure would replay the wrong verdict + # for every later repo of this visibility class, even after the API recovers. + if ! hdr=$(gh_with_backoff api -i "repos/$repo/labels"); then + printf '%s\n' "__PROBE_FAILED__" + return + fi + hdr=$(printf '%s' "$hdr" | tr -d '\r') + line=$(printf '%s\n' "$hdr" | grep -i '^X-Accepted-OAuth-Scopes:' | head -1) + value="${line#*:}" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + + local accepted + accepted=$(printf '%s' "$value" | tr ',' '\n' | while IFS= read -r s || [ -n "$s" ]; do + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + [ -n "$s" ] && printf '%s\n' "$s" + done) + + printf '%s\n' "$accepted" > "$cache_file" + printf '%s\n' "$accepted" +} + +# can_create_labels +# Returns 0 if PAT_SCOPES intersects the given accepted-scopes list. +can_create_labels() { + local accepted="$1" s + while IFS= read -r s || [ -n "$s" ]; do + [ -n "$s" ] || continue + if printf '%s\n' "$PAT_SCOPES" | grep -qxF "$s"; then + return 0 + fi + done < +# Warning-only. Never touches scope_gate_failed. +# ----------------------------------------------------------------------------- +check_labels() { + local repo="$1" required="$2" create_flag="$3" + local existing label accepted + + # `gh label list` prints a TSV table (namedescriptioncolor); ask for + # bare names one per line so the whole-line match below is correct. + # Guard the call: on failure (repo renamed/deleted, permissions, transient + # 5xx) gh_with_backoff returns 1 with empty stdout, and an unchecked capture + # would report every required label as missing on this repo. Skip the repo's + # label check instead, so one unreachable repo is one finding, not N false gaps. + if ! existing=$(gh_with_backoff label list --repo "$repo" --json name --jq '.[].name'); then + add_finding "LABEL WARN: could not list labels on $repo (see stderr above); skipping its label check rather than reporting every required label as missing." + return + fi + + while IFS= read -r label || [ -n "$label" ]; do + [ -n "$label" ] || continue + if printf '%s\n' "$existing" | grep -qxF "$label"; then + continue + fi + + accepted=$(probe_accepted_scopes "$repo") + + # probe_accepted_scopes runs in a $(...) subshell, so it signals a failed + # lookup via a stdout sentinel rather than add_finding (which would be lost + # with the subshell). Surface the finding here, in the parent. In both cases + # do not guess "cannot create" -- say we could not determine it. + case "$accepted" in + __VIS_FAILED__) + add_finding "LABEL WARN: could not determine visibility of $repo (see stderr above), so could not check whether '$label' can be created there; assuming nothing and skipping. Re-run once the API is reachable." + continue + ;; + __PROBE_FAILED__) + add_finding "LABEL WARN: '$label' missing on $repo, but could not determine whether the PAT can create it (the accepted-scope probe on $repo failed; see stderr above). Re-run once the API is reachable, or create the label manually if it stays missing." + continue + ;; + esac + + if can_create_labels "$accepted"; then + if [ "$create_flag" -eq 1 ]; then + if gh_with_backoff label create "$label" --repo "$repo" >/dev/null 2>&1; then + add_finding "LABEL created: '$label' on $repo" + else + add_finding "LABEL WARN: attempted to create '$label' on $repo but the create call failed; run: gh label create \"$label\" --repo \"$repo\"" + fi + else + add_finding "LABEL WARN: '$label' missing on $repo. PAT can create it. Run: gh label create \"$label\" --repo \"$repo\"" + fi + else + add_finding "LABEL WARN: '$label' missing on $repo. This program can launch but may not be useful without it. PAT's scopes do not cover label creation on this repo. Remediate with (a) a wider-scoped PAT, or (b) create it by hand via the GitHub web UI (Settings -> Labels) for $repo." + fi + done <&2 + return 1 + fi + ;; + --create-missing-labels) + create_flag=1 + shift + ;; + --help) + usage + return 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage >&2 + return 1 + ;; + esac + done + + if [ -z "$program" ]; then + echo "ERROR: --program is required" >&2 + usage >&2 + return 1 + fi + + local programs_dir program_file + programs_dir="${REPOMAN_PROGRAMS_DIR:-$HOME/.repoman/programs}" + program_file="$programs_dir/$program.json" + + if [ ! -f "$program_file" ]; then + echo "ERROR: program file not found: $program_file" >&2 + return 1 + fi + + local pat_scopes labels_required + pat_scopes=$(jq -r '.pat_scopes // [] | .[]' "$program_file") + labels_required=$(jq -r '.labels_required // [] | .[]' "$program_file") + + check_scopes "$pat_scopes" + + local repos repo + repos=$(repoman_get_repos) || return 1 + + if [ -n "$labels_required" ]; then + init_probe_cache + while IFS= read -r repo || [ -n "$repo" ]; do + [ -n "$repo" ] || continue + check_labels "$repo" "$labels_required" "$create_flag" + done < --program --display-name \ + --report-path [--dry-run] [--help] + +Upserts one program's entry into an _index.json program registry. + +Options: + --index PATH the _index.json to create/update (required) + --program ID program id / object key, e.g. link-health (required) + --display-name NAME human section heading, e.g. "Link Health" (required) + --report-path PATH report dir for this program (required) + --dry-run print the resulting index to stdout, write nothing + --help print this help and exit 0 + +last_run is stamped internally (date -u); it is not an argument. +EOF +} + +INDEX="" PROGRAM="" DISPLAY_NAME="" REPORT_PATH="" DRY_RUN=0 +while [ $# -gt 0 ]; do + case "$1" in + --index) [ $# -ge 2 ] || { echo "ERROR: --index needs a value" >&2; exit 2; }; INDEX="$2"; shift 2 ;; + --program) [ $# -ge 2 ] || { echo "ERROR: --program needs a value" >&2; exit 2; }; PROGRAM="$2"; shift 2 ;; + --display-name) [ $# -ge 2 ] || { echo "ERROR: --display-name needs a value" >&2; exit 2; }; DISPLAY_NAME="$2"; shift 2 ;; + --report-path) [ $# -ge 2 ] || { echo "ERROR: --report-path needs a value" >&2; exit 2; }; REPORT_PATH="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --help) usage; exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +for pair in "index:$INDEX" "program:$PROGRAM" "display-name:$DISPLAY_NAME" "report-path:$REPORT_PATH"; do + name="${pair%%:*}"; val="${pair#*:}" + if [ -z "$val" ]; then echo "ERROR: --$name is required" >&2; usage >&2; exit 2; fi +done + +# Normalize report-path to absolute. The dashboard resolves "$report_path/latest.json" +# from its own cwd, so a relative value (e.g. dep-bump's default ./reports/dep-bump) +# would make the dashboard silently skip the program when run from a different dir. +# Canonicalize via cd+pwd when the dir exists (the real flow writes reports first); +# otherwise prefix the current dir so a yet-to-exist relative path still stores absolute. +case "$REPORT_PATH" in + /*) : ;; # already absolute + *) + if [ -d "$REPORT_PATH" ]; then + if ! REPORT_PATH=$(cd "$REPORT_PATH" && pwd); then + echo "ERROR: failed to resolve absolute report-path: $REPORT_PATH" >&2 + exit 1 + fi + else + REPORT_PATH="$(pwd)/$REPORT_PATH" + fi + ;; +esac + +LAST_RUN=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# Load the current index (empty object if absent); a present-but-invalid index +# is a hard error, never silently overwritten. +if [ -f "$INDEX" ]; then + if ! current=$(jq -c . "$INDEX" 2>/dev/null); then + echo "ERROR: existing index is not valid JSON: $INDEX" >&2 + exit 1 + fi +else + current='{}' +fi + +# Upsert the entry via jq (correct escaping; no echo-splice). +if ! new_index=$(printf '%s' "$current" | jq \ + --arg p "$PROGRAM" --arg dn "$DISPLAY_NAME" \ + --arg rp "$REPORT_PATH" --arg lr "$LAST_RUN" \ + '.[$p] = {display_name: $dn, report_path: $rp, last_run: $lr}'); then + echo "ERROR: failed to compute updated index for program $PROGRAM" >&2 + exit 1 +fi + +if [ "$DRY_RUN" -eq 1 ]; then + printf '%s\n' "$new_index" + exit 0 +fi + +# Atomic write: temp in the index's dir, then mv. +index_dir=$(dirname "$INDEX") +if ! mkdir -p "$index_dir"; then + echo "ERROR: cannot create index directory: $index_dir" >&2 + exit 1 +fi +tmp="$INDEX.tmp.$$" +if ! printf '%s\n' "$new_index" > "$tmp"; then + echo "ERROR: failed to write temp index: $tmp" >&2 + exit 1 +fi +if ! mv "$tmp" "$INDEX"; then + echo "ERROR: failed to move temp index into place: $INDEX" >&2 + rm -f "$tmp" + exit 1 +fi diff --git a/skills/repoman/scripts/repoman-requirements.sh b/skills/repoman/scripts/repoman-requirements.sh new file mode 100755 index 0000000..327eb19 --- /dev/null +++ b/skills/repoman/scripts/repoman-requirements.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Single point of parsing for the SKILL.md `### Requirements (machine-readable)` +# block. Everything downstream consumes the normalized JSON this emits; nothing +# else parses SKILL.md. +# +# ## Portability +# Targets bash 3.2+ (macOS default): no mapfile, no declare -A, no associative +# arrays. +[ -n "${_REPOMAN_REQUIREMENTS_SH_LOADED:-}" ] && return +_REPOMAN_REQUIREMENTS_SH_LOADED=1 + +# Print normalized requirements JSON for a SKILL.md file. +# Usage: repoman_parse_requirements +# Emits {"pat_scopes":[...],"labels_required":[...],"labels_applied":[...],"programs":[...]} +# Missing keys default to []. Fails loud (return 1) on a malformed block. +repoman_parse_requirements() { + local skill_md="$1" + if [ ! -f "$skill_md" ]; then + echo "ERROR: repoman_parse_requirements: file not found: $skill_md" >&2 + return 1 + fi + + local in_block=0 line key body + # Per-bullet list-splitting scratch (declared here, not inside the loop, so + # block-scoping expectations are not misled; reset per bullet at use site). + local IFS_SAVE item items + # Newline-delimited item accumulators, one per recognized key. + local pat_scopes="" labels_required="" labels_applied="" programs="" + + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + '###'[[:space:]]*[Rr]equirements*) in_block=1; continue ;; + esac + if [ "$in_block" -eq 1 ]; then + # A new heading closes the leaf block. + case "$line" in + '##'[[:space:]]*|'###'[[:space:]]*) in_block=0; continue ;; + esac + # Only consider "- key: ..." bullets; ignore blank/prose lines. + case "$line" in + '- '*': '*) + key="${line#- }"; key="${key%%:*}" + body="${line#*: }" + case "$key" in + pat_scopes|labels_required|labels_applied|programs) ;; + *) echo "ERROR: repoman_parse_requirements: unknown key '$key' in Requirements block of $skill_md (recognized keys: pat_scopes, labels_required, labels_applied, programs; the block accepts no prose bullets -- put notes outside the block)" >&2; return 1 ;; + esac + # Value must be bracketed. + case "$body" in + '['*']') ;; + *) echo "ERROR: repoman_parse_requirements: value for '$key' must be a [bracketed] list in $skill_md" >&2; return 1 ;; + esac + # Strip brackets, split on commas, trim, append non-empty items. + body="${body#[}"; body="${body%]}" + IFS_SAVE="$IFS"; items="" + IFS=',' + for item in $body; do + # trim leading/trailing whitespace + item="${item#"${item%%[![:space:]]*}"}" + item="${item%"${item##*[![:space:]]}"}" + [ -n "$item" ] && items="${items}${item}"$'\n' + done + IFS="$IFS_SAVE" + case "$key" in + pat_scopes) pat_scopes="$items" ;; + labels_required) labels_required="$items" ;; + labels_applied) labels_applied="$items" ;; + programs) programs="$items" ;; + esac + ;; + esac + fi + done < "$skill_md" + + # Build each JSON array from a newline-delimited accumulator via jq -R -s. + # No sort: the normalized-JSON contract preserves the author-declared order + # from the SKILL.md bullet. Downstream consumers test membership, not order, + # so insertion order is behaviorally safe and more faithful than alphabetizing. + _rpr_json_array() { + # stdin: newline-delimited items (possibly empty) -> compact JSON array + jq -R -s 'split("\n") | map(select(length > 0))' + } + local j_pat j_lr j_la j_prog + j_pat=$(printf '%s' "$pat_scopes" | _rpr_json_array) + j_lr=$(printf '%s' "$labels_required" | _rpr_json_array) + j_la=$(printf '%s' "$labels_applied" | _rpr_json_array) + j_prog=$(printf '%s' "$programs" | _rpr_json_array) + unset -f _rpr_json_array + + jq -n \ + --argjson pat_scopes "$j_pat" \ + --argjson labels_required "$j_lr" \ + --argjson labels_applied "$j_la" \ + --argjson programs "$j_prog" \ + '{pat_scopes:$pat_scopes, labels_required:$labels_required, labels_applied:$labels_applied, programs:$programs}' +} + +# This file is a sourced library, not a command. Executing it directly used to +# exit 0 and print nothing: a caller that ran `repoman-requirements.sh ` +# (instead of sourcing it) captured an empty string, parsed it as [], and wrote +# EMPTY requirements -- a silent capability-gate failure. Fail loud instead. +# "${BASH_SOURCE[0]}" == "$0" only when this file is the executed script, not +# when it is sourced. bash 3.2-safe. +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + echo "ERROR: repoman-requirements.sh is a sourced library, not a command." >&2 + echo "Source it, then call the function:" >&2 + # Show the real resolved path to this library (BASH_SOURCE[0] is this file, + # since we only reach here when executed). A snippet with $0 or an + # unexpanded BASH_SOURCE would refer to the CALLER's script, not this one. + echo " . \"${BASH_SOURCE[0]}\"" >&2 + echo " repoman_parse_requirements " >&2 + exit 2 +fi diff --git a/skills/repoman/scripts/repoman-run.sh b/skills/repoman/scripts/repoman-run.sh new file mode 100644 index 0000000..fa66533 --- /dev/null +++ b/skills/repoman/scripts/repoman-run.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +# repoman run [target...] — the RepoMan run-path orchestrator. +# +# Pure glue: resolves a program to its step profile, (optionally) narrows to a +# named enrolled subset, runs the capability gate + clone refresh as the profile +# requires, execs the program, and writes its _index.json entry. Non-interactive +# (output routing must be set beforehand via repoman-setup.sh set-output). +# +# The run-path half of the RepoMan v0.1.0 completion work (issue #107, epic #72). +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/program-lib.sh +. "$SCRIPT_DIR/program-lib.sh" + +usage() { + # Program list is DERIVED from the table (via _known_programs) so it cannot + # drift from what is actually runnable. + cat < [target...] [--dry-run] [--help] + +Run a RepoMan program against the enrolled repos (or a named subset). + is one of: $(_known_programs) +[target...] are owner/name slugs or bare names ALREADY reconciled by the skill. +Non-interactive: output routing must be set beforehand (repoman-setup.sh set-output). +EOF +} + +# Per-program step table. One row per program. Fields (tab-separated): +# 1 program 2 profile 3 needs_clone_refresh 4 posts_output 5 writes_index +# 6 scanner_script 7 display_name 8 report_path 9 supports_dry_run +# 10 needs_live +# needs_live=yes means the program PREVIEWS by default and publishes only on an +# explicit --live (e.g. automation-health-dashboard); the orchestrator forwards +# --live on a normal run for those. The scanners push by default (silenced by +# --dry-run), so they are needs_live=no. +# Must stay consistent with KNOWN_PROGRAMS in repoman-setup.sh (unify = #76). +_PROGRAM_TABLE="\ +link-health scanner yes yes yes link-health-scanner.sh Link Health $HOME/reports/link-health yes no +dep-bump scanner yes yes yes dep-bump-scanner.sh Dependency Bumps $HOME/reports/dep-bump yes no +pr-review scanner no no no pr-review-scanner.sh PR Review yes no +weekly-report scanner no no no weekly-report.sh Weekly Report no no +automation-health index-consumer no no no automation-health-dashboard.sh Automation Health yes yes +repo-sync sync no no no repo-sync.sh Repo Sync yes no" + +# _program_row -> echoes the tab-separated row, or returns 1 if unknown. +# NOTE: the `while` runs in a subshell (right side of a pipe), so a `return` +# inside it cannot set this function's status; the trailing `grep .` detects +# whether any row matched and sets the exit status. bash 3.2-safe. +_program_row() { + local want="$1" + printf '%s\n' "$_PROGRAM_TABLE" | awk -F'\t' -v w="$want" '$1==w {print; exit}' | grep . +} + +# _known_programs -> space-separated list of known program ids. +_known_programs() { + printf '%s\n' "$_PROGRAM_TABLE" | awk -F'\t' 'NF {printf "%s ", $1}' +} + +# _row_field -> the n-th tab-separated field (1-indexed). +_row_field() { printf '%s\n' "$1" | awk -F'\t' -v n="$2" '{print $n}'; } + +# _resolve_targets -> newline-delimited owner/name set, or non-zero +# on an unknown/unenrolled target. Trusts the skill's reconciliation; only +# VALIDATES enrollment (defensive). Bare name -> repoman_match_name. With no +# targets, emits nothing (caller treats as the whole enrolled set). +_resolve_targets() { + [ "$#" -gt 0 ] || return 0 + local t resolved nmatch + for t in "$@"; do + case "$t" in + */*) + if is_enrolled "$t"; then + printf '%s\n' "$t" + else + echo "ERROR: target '$t' is not enrolled" >&2; return 1 + fi ;; + *) + if ! resolved=$(repoman_match_name "$t"); then + echo "ERROR: failed resolving target '$t'" >&2; return 1 + fi + if [ -z "$resolved" ]; then + echo "ERROR: target '$t' matches no enrolled repo" >&2; return 1 + fi + # repoman_match_name emits one line per matching owner/name. A bare name + # enrolled under >=2 owners is AMBIGUOUS: fail loud with the candidates + # rather than silently broaden the run to all of them. (This CLI accepts + # bare names directly, so it must not guess which owner was meant.) + nmatch=$(printf '%s\n' "$resolved" | grep -c .) + if [ "$nmatch" -gt 1 ]; then + echo "ERROR: target '$t' is ambiguous — enrolled under multiple owners:" >&2 + printf ' %s\n' "$resolved" >&2 + echo "Re-run with an explicit owner/name." >&2 + return 1 + fi + printf '%s\n' "$resolved" ;; + esac + done | awk 'NF && !seen[$0]++' # dedup, preserve first-seen order +} + +# Run-scoped temp dir for subset projection; cleaned on any exit (success, +# error, or signal). Only a hard kill -9 leaks, and $TMPDIR is OS-reclaimed. +RUN_TMPDIR="" +_cleanup_run_tmp() { [ -n "$RUN_TMPDIR" ] && rm -rf "$RUN_TMPDIR"; } +trap _cleanup_run_tmp EXIT + +# _project_subset -> path to a throwaway +# repos.json (inside ) holding exactly that subset. Lives in $TMPDIR (NOT +# under ~/.repoman/); written directly as a projection, which is allowed because +# it is outside the guarded dir (HARD INVARIANT guards only ~/.repoman/ state). +# The CALLER owns the dir (and its cleanup) so the EXIT trap can see it — a dir +# created in this command-substitution subshell would not be visible to the +# parent's trap, leaking the temp dir. +_project_subset() { + local dir="$1" set_in="$2" + local out="$dir/repos.json" + if ! printf '%s\n' "$set_in" \ + | jq -R 'select(length>0) | {owner: (split("/")[0]), name: (split("/")[1])}' \ + | jq -s '.' > "$out"; then + echo "ERROR: failed writing subset repos file" >&2; return 1 + fi + printf '%s\n' "$out" +} + +# _bin -> path to a sibling script, overridable by REPOMAN_RUN_BIN (tests +# inject stubs there). +_bin() { echo "${REPOMAN_RUN_BIN:-$SCRIPT_DIR}/$1"; } + +# _require_output_repo -> rc 0 if output_repo is present and +# well-formed in programs/.json (mode:same, or mode:central with a +# non-empty owner/name repo); non-zero + fail-loud otherwise. Never prompts. +_require_output_repo() { + local program="$1" progdir f mode repo + progdir="${REPOMAN_PROGRAMS_DIR:-$HOME/.repoman/programs}" + f="$progdir/$program.json" + if [ ! -f "$f" ]; then + echo "ERROR: $program: output destination not set (no $f). Run /repoman or repoman-setup.sh set-output." >&2 + return 1 + fi + mode=$(jq -r '.output_repo.mode // empty' "$f" 2>/dev/null) + if [ -z "$mode" ]; then + echo "ERROR: $program: output destination not set. Run /repoman or repoman-setup.sh set-output." >&2 + return 1 + fi + if [ "$mode" = "central" ]; then + repo=$(jq -r '.output_repo.repo // empty' "$f" 2>/dev/null) + case "$repo" in + */*) : ;; + *) echo "ERROR: $program: output_repo mode=central but 'repo' missing/malformed in $f" >&2; return 1 ;; + esac + fi + return 0 +} + +# _run_capability_check -> non-zero if the scope gate failed. +# repoman-check.sh exits non-zero iff a PAT scope gap was found; label gaps are +# warnings there and do not flip its exit. We surface its stderr and propagate. +_run_capability_check() { + local program="$1" + if ! bash "$(_bin repoman-check.sh)" --program "$program"; then + echo "ERROR: $program: capability check failed (scope gate)." >&2 + return 1 + fi +} + +# _run_clone_refresh -> non-zero only on TOTAL sync failure. repo-sync.sh exits +# 1 iff ALL enrolled repos failed (a partial failure is logged there and exits +# 0); total failure is fatal because there is nothing cloned to scan. +_run_clone_refresh() { + if ! bash "$(_bin repo-sync.sh)"; then + echo "ERROR: clone refresh failed for all repos; cannot scan." >&2 + return 1 + fi +} + +# _write_index -> registers the program +# in _index.json via repoman-index.sh. The orchestrator is the SOLE index +# writer (scanners no longer self-register). A failed index write is logged, not +# fatal: the program's reports are the primary product. +_write_index() { + local program="$1" display="$2" report_path="$3" + local index_file="${REPOMAN_INDEX_FILE:-$(dirname "$report_path")/_index.json}" + if ! bash "$(_bin repoman-index.sh)" --index "$index_file" \ + --program "$program" --display-name "$display" --report-path "$report_path"; then + echo "WARNING: failed to update program registry at $index_file (program output was still produced)" >&2 + fi +} + +main() { + local program="" dry_run=false + local -a targets=() + while [ $# -gt 0 ]; do + case "$1" in + --help|-h) usage; return 0 ;; + --dry-run) dry_run=true; shift ;; + -*) echo "ERROR: unknown flag: $1" >&2; usage >&2; return 2 ;; + *) if [ -z "$program" ]; then program="$1"; else targets+=("$1"); fi; shift ;; + esac + done + [ -n "$program" ] || { echo "ERROR: no program given" >&2; usage >&2; return 2; } + + local row + if ! row=$(_program_row "$program"); then + echo "ERROR: unknown program '$program'. Known programs: $(_known_programs)" >&2 + return 2 + fi + + local profile; profile=$(_row_field "$row" 2) + + local needs_clone; needs_clone=$(_row_field "$row" 3) + local posts_output; posts_output=$(_row_field "$row" 4) + local writes_index; writes_index=$(_row_field "$row" 5) + local scanner_script; scanner_script=$(_row_field "$row" 6) + local display_name; display_name=$(_row_field "$row" 7) + local report_path; report_path=$(_row_field "$row" 8) + local supports_dry_run; supports_dry_run=$(_row_field "$row" 9) + local needs_live; needs_live=$(_row_field "$row" 10) + + # Validate / resolve any named targets against the enrolled set (defensive; + # the skill has already reconciled them). Empty -> whole enrolled set. + local resolved_set + if ! resolved_set=$(_resolve_targets "${targets[@]+"${targets[@]}"}"); then + return 1 + fi + + # Output routing must be set beforehand for programs that post results. + if [ "$posts_output" = "yes" ]; then + _require_output_repo "$program" || return 1 + fi + + # Narrow to a subset via a throwaway repos file when targets were named. + # Only the scanner profile narrows to a subset; index-consumer (dashboard) + # and sync (repo-sync) act over the whole enrolled set regardless of targets. + if [ "$profile" = "scanner" ] && [ -n "$resolved_set" ]; then + # Create the temp dir in the PARENT so the EXIT trap (which reads the + # global RUN_TMPDIR) can clean it; a dir made inside _project_subset's + # command-substitution subshell would be invisible to the trap and leak. + if ! RUN_TMPDIR=$(mktemp -d); then + echo "ERROR: mktemp -d failed; cannot project repo subset" >&2; return 1 + fi + local subset_file + if ! subset_file=$(_project_subset "$RUN_TMPDIR" "$resolved_set"); then + return 1 + fi + export REPOMAN_REPOS_FILE="$subset_file" + # Explicit subset signal for stateful scanners: a narrowed run must scope its + # diff/close/write to only these repos and carry the rest of the previous + # report forward, and must NOT append a (partial) history row. Newline- + # delimited owner/name. Absent => whole-set run (scanners behave as before). + export REPOMAN_SUBSET_REPOS="$resolved_set" + fi + + # Execution preconditions apply to the scanner profile only. index-consumer + # (dashboard) reads the index and needs no scope/clone; sync (repo-sync) IS the + # clone refresh and takes no PAT scopes — both skip the gate and refresh. + if [ "$profile" = "scanner" ]; then + _run_capability_check "$program" || return 1 + if [ "$needs_clone" = "yes" ]; then + _run_clone_refresh || return 1 + fi + fi + + # Exec the program. Flag forwarding: + # --dry-run run: forward --dry-run to programs that accept it, so they + # suppress their mutating tail (issue/PR/docs push). + # normal run: for a needs_live program (previews by default, publishes only + # on --live, e.g. the dashboard), forward --live so a scheduled run + # actually publishes. The push-by-default scanners get nothing. + # Never forward both --dry-run and --live. + # Invoke via an explicit `bash` interpreter rather than relying on the file's + # execute bit: the repo's scanners are not uniformly chmod +x (e.g. + # dep-bump-scanner.sh is 100644 in git) and the house convention everywhere + # else is `bash