diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14337a4..c336539 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,21 +39,48 @@ jobs: pytest -q --cov=shimkit \ --cov-report=term \ --cov-report=xml:coverage.xml + # Ubuntu / Python 3.12 is the canonical coverage row: it writes the job + # summary, keeps coverage.xml, and uploads to Codecov. macOS and the + # other Pythons produce identical coverage modulo platform-gated tests. + - name: Coverage summary + # always(): show the numbers even when the run fails the floor. + if: >- + always() && matrix.os == 'ubuntu-latest' && matrix.python == '3.12' + && hashFiles('coverage.xml') != '' + run: python scripts/coverage_summary.py + - name: Keep coverage.xml + if: >- + always() && matrix.os == 'ubuntu-latest' && matrix.python == '3.12' + && hashFiles('coverage.xml') != '' + uses: actions/upload-artifact@v7 + with: + name: coverage-xml-${{ github.sha }} + path: coverage.xml + retention-days: 7 - name: Upload coverage to Codecov - # Only upload from one matrix cell to avoid double-counting. - # Ubuntu 3.12 is the canonical row; macOS / other Pythons - # produce identical coverage modulo platform-gated tests. + id: codecov if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12' + # Advisory: the gate is fail_under, and the job summary shows the + # numbers. fail_ci_if_error makes the CLI exit non-zero when Codecov + # answers HTTP >= 400 (with it off, the action always exits 0 and a + # rejection is silent); continue-on-error keeps that from failing the + # job, and the next step turns it into a visible warning. + continue-on-error: true uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: ./coverage.xml flags: unit disable_search: true # Uses CODECOV_TOKEN when one is configured; otherwise a tokenless - # upload. Codecov being down or refusing the upload never blocks a - # PR: the local fail_under floor is the gate. + # upload. token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false + fail_ci_if_error: true + - name: Report a rejected Codecov upload + if: steps.codecov.outcome == 'failure' + run: | + msg="Codecov rejected the upload (repository not activated or token mismatch) — coverage is still enforced by fail_under and shown in the job summary. See the 'Upload coverage to Codecov' step log for the server response." + echo "::warning title=Codecov upload failed::$msg" + printf '\n> [!WARNING]\n> %s\n' "$msg" >> "$GITHUB_STEP_SUMMARY" security: runs-on: ubuntu-latest diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 92cba03..91bd161 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -36,11 +36,31 @@ jobs: pytest -q --cov=shimkit \ --cov-report=term \ --cov-report=xml:coverage.xml + - name: Coverage summary + if: always() && hashFiles('coverage.xml') != '' + run: python scripts/coverage_summary.py + - name: Keep coverage.xml + if: always() && hashFiles('coverage.xml') != '' + uses: actions/upload-artifact@v7 + with: + name: coverage-xml-${{ github.sha }} + path: coverage.xml + retention-days: 7 - name: Upload coverage to Codecov + id: codecov + # Same detection as ci.yml: fail_ci_if_error surfaces an HTTP >= 400 + # as a step failure, continue-on-error keeps the job green. + continue-on-error: true uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: ./coverage.xml flags: unit disable_search: true token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false + fail_ci_if_error: true + - name: Report a rejected Codecov upload + if: steps.codecov.outcome == 'failure' + run: | + msg="Codecov rejected the upload (repository not activated or token mismatch) — coverage is still enforced by fail_under and shown in the job summary. See the 'Upload coverage to Codecov' step log for the server response." + echo "::warning title=Codecov upload failed::$msg" + printf '\n> [!WARNING]\n> %s\n' "$msg" >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 02afcb0..d591057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +### Added + +- Every CI run writes a coverage summary to its job summary (total, + floor, lowest-covered files, full table) from `coverage.xml`, via the + stdlib-only `scripts/coverage_summary.py`. `coverage.xml` is kept as a + run artifact for seven days. Both `ci.yml` and `coverage-baseline.yml` + do this, so coverage is visible without Codecov being configured. + ### Changed - CI runs on `pull_request` and `workflow_dispatch` only; the @@ -22,6 +30,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Fixed +- A rejected Codecov upload was silent: with `fail_ci_if_error: false` + the action exits 0 whatever the server answers. The step now runs with + `fail_ci_if_error: true` under `continue-on-error`, and a failed + outcome raises a `Codecov upload failed` warning annotation. - The Codecov upload step never ran. Its condition read `matrix.python-version`, but the matrix key is `python`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81ddbc4..b082100 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -126,9 +126,22 @@ A tool joins shimkit only if it shares ≥2 of `Platform` / `Shell` / `pyproject.toml`; plain `pytest --cov=shimkit` fails below it, locally and in CI. It is the measured total rounded down. Raise it when coverage rises; never lower it to get a PR through. -- Every PR uploads coverage to Codecov from the Ubuntu / Python 3.12 CI - cell. The upload is advisory (`fail_ci_if_error: false`), so a Codecov - outage never blocks a merge. +- Coverage does not depend on Codecov. The Ubuntu / Python 3.12 CI job + writes a summary to its job summary page (Actions → the run → the + `test (ubuntu-latest, 3.12)` job → Summary): the total, whether it meets + the floor, and the ten lowest-covered files, with the full + `coverage report --format=markdown` table folded underneath. It is + rendered by `scripts/coverage_summary.py` from `coverage.xml`, which is + also kept as the `coverage-xml-` run artifact for seven days. Run + the script locally after `pytest --cov=shimkit --cov-report=xml` to see + the same table on stdout. +- That job also uploads to Codecov. The upload is advisory: the step runs + with `continue-on-error`, so a Codecov outage never blocks a merge. It + runs with `fail_ci_if_error: true` so a rejected upload (HTTP 4xx, e.g. + repository not activated on codecov.io, or a `CODECOV_TOKEN` that does + not match the repository) fails the step rather than passing silently, + and the next step turns that into a `Codecov upload failed` warning + annotation on the run. - `main` has no push trigger. `coverage-baseline.yml` refreshes the Codecov baseline every Monday (06:00 America/New_York) and on demand (`gh workflow run coverage-baseline.yml`). `codecov.yml` turns on diff --git a/README.md b/README.md index 16c6806..3b49340 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,9 @@ mypy src/shimkit ``` CI runs the same four commands on macOS + Ubuntu × Python 3.10/3.11/3.12/3.13. +Coverage for each run (total, the enforced floor, and the lowest-covered files) +is in the run's job summary for the Ubuntu / Python 3.12 job, with `coverage.xml` +kept as a run artifact for seven days. See [`CONTRIBUTING.md`](CONTRIBUTING.md#coverage). ## License diff --git a/scripts/coverage_summary.py b/scripts/coverage_summary.py new file mode 100644 index 0000000..7bcaa6d --- /dev/null +++ b/scripts/coverage_summary.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Render a Markdown coverage summary from ``coverage.xml``. + +CI appends the output to ``$GITHUB_STEP_SUMMARY`` so every run shows its +coverage on the run page, whether or not Codecov accepted the upload. It +uses only the standard library and the ``coverage`` package that +``pytest-cov`` already installs, so no third-party action is involved. + +Usage:: + + python scripts/coverage_summary.py [--xml coverage.xml] [--lowest 10] + +Writes to ``$GITHUB_STEP_SUMMARY`` when that is set (appending, as GitHub +expects), otherwise to stdout. Exits non-zero when the XML is missing or +lists no files, so a report that measured nothing is never rendered as a +clean summary. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class FileCoverage: + path: str + statements: int + covered: int + + @property + def missed(self) -> int: + return self.statements - self.covered + + @property + def percent(self) -> float: + return 100.0 if self.statements == 0 else 100.0 * self.covered / self.statements + + +def parse_coverage_xml(path: Path) -> list[FileCoverage]: + """Return one entry per source file in a Cobertura ``coverage.xml``.""" + root = ET.parse(path).getroot() + files: dict[str, FileCoverage] = {} + for cls in root.iter("class"): + filename = cls.get("filename", "") + lines = cls.findall("./lines/line") + covered = sum(1 for line in lines if int(line.get("hits", "0")) > 0) + prev = files.get(filename) + if prev is not None: # coverage.py emits one per file, but be safe + files[filename] = FileCoverage( + filename, prev.statements + len(lines), prev.covered + covered + ) + else: + files[filename] = FileCoverage(filename, len(lines), covered) + return sorted(files.values(), key=lambda f: f.path) + + +def configured_floor() -> float | None: + """The ``fail_under`` coverage.py itself enforces, read through its own config.""" + try: + import coverage + except ImportError: + return None + value = coverage.Coverage().get_option("report:fail_under") + return float(value) if value else None + + +def full_report() -> str | None: + """``coverage report --format=markdown`` (coverage.py >= 7.0), if the data file exists.""" + if not Path(".coverage").exists(): + return None + # Exit status 2 means "below fail_under"; the table is still complete. + proc = subprocess.run( # fixed argv, no shell + [sys.executable, "-m", "coverage", "report", "--format=markdown"], + capture_output=True, + text=True, + check=False, + ) + return proc.stdout.strip() or None + + +def render(files: list[FileCoverage], floor: float | None, lowest: int, full: str | None) -> str: + statements = sum(f.statements for f in files) + covered = sum(f.covered for f in files) + total = 100.0 if statements == 0 else 100.0 * covered / statements + + out = ["## Coverage", ""] + if floor is None: + out.append(f"**Total: {total:.2f}%** ({covered}/{statements} statements, no floor set)") + else: + verdict = "meets" if total >= floor else "**below**" + out.append( + f"**Total: {total:.2f}%** ({covered}/{statements} statements) " + f"{verdict} the enforced floor of **{floor:g}%** " + "(`[tool.coverage.report] fail_under`)." + ) + out += [ + "", + f"Measured across {len(files)} files. Lowest-covered:", + "", + "| File | Stmts | Miss | Cover |", + "|---|---:|---:|---:|", + ] + for f in sorted(files, key=lambda f: (f.percent, -f.missed))[:lowest]: + out.append(f"| `{f.path}` | {f.statements} | {f.missed} | {f.percent:.1f}% |") + if full: + out += ["", "
Full report", "", full, "", "
"] + out.append("") + return "\n".join(out) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--xml", type=Path, default=Path("coverage.xml")) + parser.add_argument("--lowest", type=int, default=10) + args = parser.parse_args(argv) + + if not args.xml.is_file(): + print(f"::error::{args.xml} not found; nothing to summarise", file=sys.stderr) + return 1 + files = parse_coverage_xml(args.xml) + if not files: + print( + f"::error::{args.xml} lists no files; refusing to report 0 as a result", file=sys.stderr + ) + return 1 + + text = render(files, configured_floor(), args.lowest, full_report()) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as fh: + fh.write(text) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_coverage_summary.py b/tests/test_coverage_summary.py new file mode 100644 index 0000000..52fcf54 --- /dev/null +++ b/tests/test_coverage_summary.py @@ -0,0 +1,74 @@ +"""Tests for scripts/coverage_summary.py, the CI job-summary renderer.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "coverage_summary.py" +_spec = importlib.util.spec_from_file_location("coverage_summary", _SCRIPT) +assert _spec is not None and _spec.loader is not None +cs = importlib.util.module_from_spec(_spec) +sys.modules["coverage_summary"] = cs +_spec.loader.exec_module(cs) + +_XML = """ + + + + + + + + + + + +""" + + +def _write(tmp_path: Path, body: str) -> Path: + p = tmp_path / "coverage.xml" + p.write_text(body, encoding="utf-8") + return p + + +def test_parse_counts_statements_and_hits(tmp_path: Path) -> None: + files = cs.parse_coverage_xml(_write(tmp_path, _XML)) + assert [(f.path, f.statements, f.covered) for f in files] == [ + ("src/shimkit/a.py", 2, 2), + ("src/shimkit/b.py", 3, 1), + ] + + +def test_render_lists_lowest_first_and_compares_to_floor(tmp_path: Path) -> None: + files = cs.parse_coverage_xml(_write(tmp_path, _XML)) + text = cs.render(files, floor=84.0, lowest=10, full=None) + assert "**Total: 60.00%** (3/5 statements) **below** the enforced floor of **84%**" in text + assert "Measured across 2 files" in text + assert text.index("src/shimkit/b.py") < text.index("src/shimkit/a.py") + assert cs.render(files, floor=50.0, lowest=1, full=None).count("| `src/") == 1 + + +def test_main_appends_to_step_summary(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + summary = tmp_path / "summary.md" + summary.write_text("earlier step\n", encoding="utf-8") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + monkeypatch.chdir(tmp_path) # no .coverage here, so no full report + assert cs.main(["--xml", str(_write(tmp_path, _XML))]) == 0 + text = summary.read_text(encoding="utf-8") + assert text.startswith("earlier step\n## Coverage") + + +@pytest.mark.parametrize("body", [None, ""]) +def test_main_refuses_missing_or_empty_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str | None +) -> None: + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + xml = tmp_path / "coverage.xml" + if body is not None: + xml.write_text(body, encoding="utf-8") + assert cs.main(["--xml", str(xml)]) == 1