feat: renovate, a branded verdict header and comment: on-failure

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
           working-directory: test/fixtures/nuxt-app
           min-score: 40
           baseline: none
-          comment-key: single
+          comment: on-failure
       - name: Outputs are the fixture's numbers
         env:
           OUTPUTS: ${{ toJSON(steps.map.outputs) }}
@@ -50,7 +50,7 @@ jobs:
             test/fixtures/*
           limit: 3
           baseline: none
-          comment-key: monorepo
+          comment: false
       - name: Every fixture package is scanned, the non-package is not
         env:
           OUTPUTS: ${{ toJSON(steps.map.outputs) }}
@@ -131,7 +131,6 @@ jobs:
         with:
           packages: |
             test/fixtures/*
-          comment-key: baseline
       - name: The base was scanned and compared
         env:
           OUTPUTS: ${{ toJSON(steps.map.outputs) }}

AGENTS.md

@@ -27,6 +27,10 @@ test/fixtures/        small apps the workflow scans: nuxt-app, express-app, hono
 - **Every behaviour has a job in `test.yml`.** A new input or output gets a job that exercises it against a fixture and asserts on `steps.map.outputs`. Unit tests cover the modules; the workflow is the proof the composite step works.
 - Code style follows `evloghq/evlog`'s AGENTS.md: no defensive code the surrounding file does not have, no silent fallbacks, comments only for constraints the code cannot express, plain factual prose everywhere.
 
+## Dependencies
+
+Renovate keeps two things current: the SHA-pinned actions in the workflows, and the `@evlog/cli` release pinned as the `version` default in `action.yml` (the `# renovate:` marker above it is what Renovate reads). A CLI bump PR is reviewed like any change, merged, and released with a tag, because it changes what every user's gate sees.
+
 ## Releasing
 
 Tag `vX.Y.Z` on `main`. `release.yml` moves `vX` to it. Users ride `@v1`. The `version` default in `action.yml` is the `@evlog/cli` release the action was tested with: bumping it is a change to this repository, released with a tag, never implied by a CLI release.

README.md

@@ -44,7 +44,7 @@ The judgment is the CLI's. The action adds no rule of its own, so `npx evlog map
 | `gate` | `true` | `false` reports and sets outputs without failing the step. |
 | `annotations` | `true` | Draw findings on the diff. |
 | `summary` | `true` | Write the report to the job summary. |
-| `comment` | `true` | Post and update the pull request comment. Needs `pull-requests: write`; skipped with a notice when the token cannot write (forks). |
+| `comment` | `true` | `true` posts one comment and keeps it updated; `on-failure` only posts when the gate failed, and updates an existing comment either way; `false` posts nothing. Needs `pull-requests: write`; skipped with a notice when the token cannot write (forks). |
 | `comment-key` | `default` | Keeps separate comments when the action runs more than once on a pull request. |
 | `token` | `${{ github.token }}` | Token for the comment. |
 
@@ -98,6 +98,31 @@ Each action release pins the `@evlog/cli` it was tested with, so `@v1` is determ
 
 On a push there is no base to compare against, so `baseline: auto` turns the comparison off and the run reports. Pass `baseline: git:origin/main` to compare against a committed `evlog.map.json` instead.
 
+### Comment only when something is wrong
+
+```yaml
+      - uses: evloghq/action@v1
+        with:
+          comment: on-failure
+```
+
+A clean pull request gets annotations and a job summary, nothing in the conversation. The comment appears when a check regressed or the score missed `min-score`, and it is updated to the passing state once that is fixed rather than left behind.
+
+### Post as your own bot
+
+The comment is posted with the workflow's token, so it shows as `github-actions`. To post as an app of your own (an "evlog" bot with its avatar), mint a token in the workflow and pass it in:
+
+```yaml
+      - uses: actions/create-github-app-token@v2
+        id: app
+        with:
+          app-id: ${{ vars.EVLOG_APP_ID }}
+          private-key: ${{ secrets.EVLOG_APP_KEY }}
+      - uses: evloghq/action@v1
+        with:
+          token: ${{ steps.app.outputs.token }}
+```
+
 ## Frameworks
 
 Everything `evlog map` scans: Nuxt, Nitro, Next.js App Router, TanStack Start, Hono, Express, and Fastify. Detection reads `package.json`; nothing is installed.

action.yml

@@ -10,9 +10,11 @@ inputs:
     description: CLI command to run. Only `map` for now.
     default: map
   version:
-    # The CLI release this action was tested against. A release of the action
-    # moves it; a CLI release on its own never changes what a user's gate sees.
+    # The CLI release this action was tested against. Renovate proposes the
+    # bump; a release of the action ships it. A CLI release on its own never
+    # changes what a user's gate sees.
     description: '`@evlog/cli` version to run, or a full package spec (a pkg.pr.new URL). Defaults to the release this action was tested with; `latest` follows npm.'
+    # renovate: datasource=npm depName=@evlog/cli
     default: 0.8.0
   working-directory:
     description: Directory of the project to scan, relative to the workspace.
@@ -39,7 +41,7 @@ inputs:
     description: Write the report to the job summary.
     default: 'true'
   comment:
-    description: Post one comment on the pull request and keep it updated. Needs `pull-requests write`; skipped when the token cannot write.
+    description: '`true` posts one comment on the pull request and keeps it updated. `on-failure` only posts when the gate failed, and updates an existing comment either way. `false` posts nothing. Needs `pull-requests write`; skipped when the token cannot write.'
     default: 'true'
   comment-key:
     description: Distinguishes comments when the action runs more than once on a pull request.

renovate.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended",
+    "helpers:pinGitHubActionDigests"
+  ],
+  "schedule": ["before 3am on Monday"],
+  "timezone": "UTC",
+  "minimumReleaseAge": "2 days",
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "The @evlog/cli release pinned as the `version` default in action.yml",
+      "managerFilePatterns": ["/^action\\.yml$/"],
+      "matchStrings": ["# renovate: datasource=npm depName=@evlog/cli\\n\\s+default: (?<currentValue>\\S+)"],
+      "datasourceTemplate": "npm",
+      "depNameTemplate": "@evlog/cli"
+    }
+  ],
+  "packageRules": [
+    {
+      "description": "Our own release: no waiting period, and it gets its own PR so the bump is a release of the action",
+      "matchDepNames": ["@evlog/cli"],
+      "minimumReleaseAge": null,
+      "commitMessageTopic": "the pinned @evlog/cli",
+      "schedule": ["at any time"]
+    }
+  ]
+}

src/lib/comment.mjs

@@ -26,8 +26,12 @@ async function request(fetchFn, url, token, init = {}) {
  * Create or update the comment. Returns what happened so the caller can say
  * so; a token that cannot write (a fork, read-only workflow permissions) is
  * reported, not thrown: the annotations and the summary still stand.
+ *
+ * `create: false` only edits a comment that is already there, which is how a
+ * gate that failed on the last run and passes now gets its comment updated
+ * without a passing run ever starting one.
  */
-export async function upsertComment({ event, token, key, body, fetchFn = fetch }) {
+export async function upsertComment({ event, token, key, body, create = true, fetchFn = fetch }) {
   const { apiUrl, repository, pullRequest } = event
   if (!pullRequest) return { outcome: 'skipped', reason: 'not a pull request' }
   if (!token) return { outcome: 'skipped', reason: 'no token' }
@@ -39,6 +43,7 @@ export async function upsertComment({ event, token, key, body, fetchFn = fetch }
   if (list.status === 401 || list.status === 403) return { outcome: 'skipped', reason: `token cannot read comments (${list.status})` }
   if (!list.ok) return { outcome: 'failed', reason: `listing comments: ${list.status}` }
   const existing = (await list.json()).find(comment => typeof comment.body === 'string' && comment.body.startsWith(tag))
+  if (!existing && !create) return { outcome: 'skipped', reason: 'gate passed and no comment to update' }
 
   const response = existing
     ? await request(fetchFn, `${apiUrl}/repos/${repository}/issues/comments/${existing.id}`, token, { method: 'PATCH', body: JSON.stringify({ body: content }) })

src/lib/inputs.mjs

@@ -7,6 +7,7 @@
 
 const COMMANDS = ['map']
 const BASELINE_OFF = ['none', 'false', 'off', '']
+const COMMENT_MODES = { true: 'always', false: 'never', 'on-failure': 'on-failure' }
 
 export class InputError extends Error {}
 
@@ -25,6 +26,12 @@ function integer(name, raw, { min, max }) {
   return value
 }
 
+function commentMode(raw) {
+  const mode = COMMENT_MODES[raw.trim().toLowerCase() || 'true']
+  if (!mode) throw new InputError(`Input \`comment\` must be true, false or on-failure, got "${raw}"`)
+  return mode
+}
+
 /** Lines of `packages`, trimmed, comments and blanks dropped. */
 export function parsePackages(raw) {
   return raw.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'))
@@ -61,7 +68,7 @@ export function readInputs(env = process.env) {
     gate: bool('gate', get('GATE') || 'true'),
     annotations: bool('annotations', get('ANNOTATIONS') || 'true'),
     summary: bool('summary', get('SUMMARY') || 'true'),
-    comment: bool('comment', get('COMMENT') || 'true'),
+    comment: commentMode(get('COMMENT')),
     commentKey: get('COMMENT_KEY').trim() || 'default',
     token: get('TOKEN'),
   }

src/lib/report.mjs

@@ -7,6 +7,8 @@
  */
 
 const DOCS = 'https://evlog.dev'
+const LOGO = 'https://www.evlog.dev/evlog.svg'
+const ACTION = 'https://github.com/evloghq/action'
 
 export function grade(score) {
   if (score >= 90) return 'excellent'
@@ -51,13 +53,42 @@ function statusCell(result) {
   return `failed (${result.reasons.join(', ')})`
 }
 
+/**
+ * The one line to read: lowest score, how it moved, what failed.
+ *
+ * `baselineRef` names what the delta is against (`main`); without it the
+ * movement is left out rather than attributed to nothing.
+ */
+export function verdict(results, { baselineRef } = {}) {
+  const score = Math.min(...results.map(result => result.score))
+  const parts = [`**${score}** ${grade(score)}`]
+  if (results.length > 1) parts[0] += ` across ${results.length} packages`
+  const withBaseline = results.filter(result => result.baseline)
+  if (withBaseline.length > 0) {
+    const delta = Math.min(...withBaseline.map(result => result.baseline.delta))
+    const against = baselineRef ? ` against ${code(baselineRef)}` : ''
+    parts.push(delta === 0 ? `unchanged${against}` : `${signed(delta)}${against}`)
+    const regressions = withBaseline.reduce((sum, result) => sum + result.baseline.regressions.length, 0)
+    parts.push(regressions === 0 ? 'no regression' : `${regressions} regression${regressions === 1 ? '' : 's'}`)
+  }
+  const below = results.filter(result => result.reasons.some(reason => reason.startsWith('below')))
+  if (below.length > 0) parts.push(below[0].reasons.find(reason => reason.startsWith('below')).replace('--min-score', 'min-score'))
+  return parts.join(' · ')
+}
+
 /**
  * @param results one entry per package, from `main.mjs`; `name` is how the
  * package is shown, `path` where it sits in the repository, for links
- * @param context repository, sha and server for links; cliVersion for the footer
+ * @param context repository, sha and server for links; cliVersion for the
+ * footer; baselineRef for the verdict
  */
 export function renderReport(results, context) {
-  const lines = ['## evlog map', '']
+  const lines = [
+    `### <img src="${LOGO}" height="18" alt=""> evlog map`,
+    '',
+    verdict(results, context),
+    '',
+  ]
   const hasBaseline = results.some(result => result.baseline)
 
   lines.push(`| Package | Score | ${hasBaseline ? 'Δ | ' : ''}Instrumented | Partial | Dark | Gate |`)
@@ -97,6 +128,6 @@ export function renderReport(results, context) {
     lines.push('', '</details>')
   }
 
-  lines.push('', `<sub>evlog map${context.cliVersion ? ` v${context.cliVersion}` : ''} · [how the score works](${DOCS}/cli/scoring) · [what each check expects](${DOCS}/cli/rules)</sub>`)
+  lines.push('', `<sub>[evloghq/action](${ACTION}) · evlog map${context.cliVersion ? ` v${context.cliVersion}` : ''} · [how the score works](${DOCS}/cli/scoring) · [what each check expects](${DOCS}/cli/rules)</sub>`)
   return lines.join('\n')
 }

src/main.mjs

@@ -83,17 +83,19 @@ async function main() {
     base?.cleanup()
   }
 
-  const report = renderReport(results, { ...event, cliVersion })
+  const outputs = aggregate(results)
+  const baselineRef = baseline.mode === 'base' ? baseline.ref : baseline.mode === 'spec' ? baseline.spec : undefined
+  const report = renderReport(results, { ...event, cliVersion, baselineRef })
   if (inputs.summary && event.summaryFile) appendFileSync(event.summaryFile, `${report}\n`)
 
-  if (inputs.comment && event.pullRequest) {
-    const outcome = await upsertComment({ event, token: inputs.token, key: inputs.commentKey, body: report })
+  if (inputs.comment !== 'never' && event.pullRequest) {
+    const create = inputs.comment === 'always' || !outputs.passed
+    const outcome = await upsertComment({ event, token: inputs.token, key: inputs.commentKey, body: report, create })
     if (outcome.outcome === 'skipped') notice(`pull request comment skipped: ${outcome.reason}`)
     else if (outcome.outcome === 'failed') fail(`pull request comment ${outcome.reason}`)
     else log(`evlog: comment ${outcome.outcome}`)
   }
 
-  const outputs = aggregate(results)
   writeOutputs(outputs, event.outputFile)
 
   if (!outputs.passed && inputs.gate) process.exitCode = 1

test/comment.test.mjs

@@ -73,6 +73,18 @@ describe('upsertComment', () => {
     assert.deepEqual(outcome, { outcome: 'failed', reason: 'creating comment: 500' })
   })
 
+  it('with create off, updates an existing comment but never starts one', async () => {
+    const none = fakeFetch([{ method: 'GET', url: /comments/, json: [] }])
+    assert.deepEqual(await upsertComment({ event, token: 't', key: 'default', body: 'x', create: false, fetchFn: none.fetchFn }), { outcome: 'skipped', reason: 'gate passed and no comment to update' })
+    assert.equal(none.calls.length, 1)
+
+    const some = fakeFetch([
+      { method: 'GET', url: /comments/, json: [{ id: 3, body: `${marker('default')}\nold` }] },
+      { method: 'PATCH', url: /issues\/comments\/3$/, json: { id: 3 } },
+    ])
+    assert.deepEqual(await upsertComment({ event, token: 't', key: 'default', body: 'x', create: false, fetchFn: some.fetchFn }), { outcome: 'updated', id: 3 })
+  })
+
   it('skips outside a pull request and without a token', async () => {
     assert.deepEqual(await upsertComment({ event: { ...event, pullRequest: undefined }, token: 't', key: 'k', body: 'x' }), { outcome: 'skipped', reason: 'not a pull request' })
     assert.deepEqual(await upsertComment({ event, token: '', key: 'k', body: 'x' }), { outcome: 'skipped', reason: 'no token' })

test/inputs.test.mjs

@@ -18,7 +18,7 @@ describe('readInputs', () => {
       gate: true,
       annotations: true,
       summary: true,
-      comment: true,
+      comment: 'always',
       commentKey: 'default',
       token: 't',
     })
@@ -42,6 +42,12 @@ describe('readInputs', () => {
     })
   }
 
+  it('reads the comment modes', () => {
+    assert.equal(readInputs(env({ INPUT_COMMENT: 'false' })).comment, 'never')
+    assert.equal(readInputs(env({ INPUT_COMMENT: 'on-failure' })).comment, 'on-failure')
+    assert.throws(() => readInputs(env({ INPUT_COMMENT: 'sometimes' })), /`comment` must be true, false or on-failure/)
+  })
+
   it('rejects a boolean that is neither true nor false', () => {
     assert.throws(() => readInputs(env({ INPUT_GATE: 'yes' })), /`gate` must be true or false/)
   })

test/report.test.mjs

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict'
 import { describe, it } from 'node:test'
-import { grade, prioritize, renderReport } from '../src/lib/report.mjs'
+import { grade, prioritize, renderReport, verdict } from '../src/lib/report.mjs'
 
 const route = (path, { score = 100, checks = {}, sensitivity = 'none', file = 'src/x.ts', line = 1, id = path } = {}) => ({
   id, method: 'GET', path, file, handler: { line }, score, sensitivity: { level: sensitivity, reasons: [] },
@@ -35,16 +35,37 @@ describe('prioritize', () => {
   })
 })
 
+describe('verdict', () => {
+  it('is the score alone without a baseline', () => {
+    assert.equal(verdict([result()]), '**82** good')
+  })
+
+  it('names the movement, the ref and the regressions with a baseline', () => {
+    const r = result({ score: 56, baseline: { delta: -8, regressions: [{}], fixed: [] } })
+    assert.equal(verdict([r], { baselineRef: 'main' }), '**56** needs work · -8 against `main` · 1 regression')
+    const ok = result({ baseline: { delta: 0, regressions: [], fixed: [] } })
+    assert.equal(verdict([ok], { baselineRef: 'main' }), '**82** good · unchanged against `main` · no regression')
+  })
+
+  it('takes the lowest package and counts them', () => {
+    assert.equal(verdict([result(), result({ name: 'apps/api', score: 61 })]), '**61** needs work across 2 packages')
+  })
+
+  it('says when the threshold was missed', () => {
+    assert.equal(verdict([result({ score: 70, reasons: ['below --min-score 80'] })]), '**70** needs work · below min-score 80')
+  })
+})
+
 describe('renderReport', () => {
   it('writes one row per package and no delta column without a baseline', () => {
     const md = renderReport([result(), result({ name: 'apps/api', framework: 'hono', score: 61 })], context)
-    assert.match(md, /^## evlog map/)
+    assert.match(md, /^### <img src="https:\/\/www\.evlog\.dev\/evlog\.svg" height="18" alt=""> evlog map\n\n\*\*61\*\* needs work across 2 packages\n/)
+    assert.match(md, /<sub>\[evloghq\/action\]\(https:\/\/github\.com\/evloghq\/action\) · evlog map v0\.8\.0/)
     assert.match(md, /\| Package \| Score \| Instrumented \| Partial \| Dark \| Gate \|/)
     assert.match(md, /\| `shop` \(nuxt\) \| \*\*82\*\* good \| 3 \| 1 \| 1 \| passed \|/)
     assert.match(md, /\| `apps\/api` \(hono\) \| \*\*61\*\* needs work /)
     assert.doesNotMatch(md, /Δ/)
     assert.doesNotMatch(md, /### Regressions/)
-    assert.match(md, /evlog map v0\.8\.0/)
   })
 
   it('adds the delta column, the regressions and the fixes with a baseline', () => {
@@ -88,6 +109,6 @@ describe('renderReport', () => {
     const md = renderReport([result({ routes: [route('/health', { score: 0, checks: { 'wide-event': 'fail' } })] })], { cliVersion: undefined })
     assert.match(md, /`src\/x\.ts:1`: `wide-event`/)
     assert.doesNotMatch(md, /github\.com\/acme/)
-    assert.match(md, /<sub>evlog map · /)
+    assert.match(md, /· evlog map · \[how/)
   })
 })