From 782c4d85ab84d3897e7b769b31a627bd8c934eb2 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 5 Oct 2026 20:23:55 -0500 Subject: [PATCH 1/2] ci: make Codecov reporting informational --- .github/workflows/check.yml | 53 ++-------- codecov.yml | 8 ++ tools/tests/test_workflow_contracts.py | 134 +++++++++---------------- 3 files changed, 68 insertions(+), 127 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5a91b97d1..7a34a887c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -286,6 +286,8 @@ jobs: if-no-files-found: error retention-days: 14 - name: Upload release-tool coverage + continue-on-error: true + timeout-minutes: 2 uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: disable_search: true @@ -294,6 +296,8 @@ jobs: flags: release-tools use_oidc: true - name: Upload libatrinik coverage + continue-on-error: true + timeout-minutes: 2 uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: disable_search: true @@ -824,6 +828,8 @@ jobs: if-no-files-found: error retention-days: 14 - name: Upload coverage + continue-on-error: true + timeout-minutes: 2 uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: override_commit: ${{ env.COVERAGE_SHA }} @@ -924,6 +930,8 @@ jobs: "${CLASSIC_LINUX_IMAGE}" \ "${CLASSIC_GPU_COVERAGE_IMAGE}" - name: Upload coverage + continue-on-error: true + timeout-minutes: 2 uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: override_commit: ${{ env.COVERAGE_SHA }} @@ -982,55 +990,16 @@ jobs: "${CLASSIC_LINUX_IMAGE}" \ "${CLASSIC_GPU_COVERAGE_IMAGE}" - name: Upload GPU-complete client coverage + continue-on-error: true + timeout-minutes: 2 uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: override_commit: ${{ env.COVERAGE_SHA }} disable_search: true - fail_ci_if_error: true + fail_ci_if_error: false files: client/coverage.xml flags: client-gpu use_oidc: true - - name: Wait for complete client coverage processing - env: - CODECOV_REPORT_API: >- - https://api.codecov.io/api/v2/github/${{ github.repository_owner }}/repos/${{ github.event.repository.name }}/totals/ - run: | - set -euo pipefail - for attempt in {1..90}; do - reports_ready=true - for flag in client-unit client-gpu; do - if response=$( - curl --fail --silent --show-error --get \ - --data-urlencode "sha=${COVERAGE_SHA}" \ - --data-urlencode "flag=${flag}" \ - "${CODECOV_REPORT_API}" - ); then - if ! sessions=$(printf '%s' "${response}" | jq -er '.totals.sessions // 0') || - [ "${sessions}" -lt 1 ]; then - reports_ready=false - break - fi - else - curl_status=$? - case "${curl_status}" in - 35|51|58|60|77) - echo "Codecov TLS transfer failed (curl exit ${curl_status}); this is not a coverage processing delay." >&2 - exit "${curl_status}" - ;; - esac - reports_ready=false - break - fi - done - if [ "${reports_ready}" = true ]; then - echo "Codecov processed both client coverage reports." - exit 0 - fi - echo "Waiting for Codecov client coverage processing (${attempt}/90)..." - sleep 10 - done - echo "Codecov did not process both client coverage reports within 15 minutes." >&2 - exit 1 integrated: name: Integrated client/server graph diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..bfdc9877d --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true diff --git a/tools/tests/test_workflow_contracts.py b/tools/tests/test_workflow_contracts.py index 76160787e..ad494cc5b 100644 --- a/tools/tests/test_workflow_contracts.py +++ b/tools/tests/test_workflow_contracts.py @@ -1844,14 +1844,13 @@ def test_linux_checks_pin_image_and_isolate_compiler_caches(self) -> None: self.assertIn("name: classic-gpu-shaders", client) self.assertIn("name: classic-gpu-shaders", gpu_coverage_job) self.assertIn("name: classic-gpu-shaders", integrated) - self.assertIn( - "name: Wait for complete client coverage processing", - gpu_coverage_job, + self.assertNotIn( + "Wait for complete client coverage processing", gpu_coverage_job ) - self.assertIn("api.codecov.io/api/v2/github/", gpu_coverage_job) + self.assertNotIn("api.codecov.io", gpu_coverage_job) self.assertIn("flags: client-unit", client) self.assertIn("flags: client-gpu", gpu_coverage_job) - self.assertIn("fail_ci_if_error: true", gpu_coverage_job) + self.assertIn("fail_ci_if_error: false", gpu_coverage_job) self.assertLess(gpu_coverage_job.index("name: Build and exercise complete GPU coverage"), gpu_coverage_job.index("name: Upload GPU-complete client coverage")) self.assertIn( @@ -1862,24 +1861,15 @@ def test_linux_checks_pin_image_and_isolate_compiler_caches(self) -> None: self.assertIn("override_commit: ${{ env.COVERAGE_SHA }}", client) self.assertIn("override_commit: ${{ env.COVERAGE_SHA }}", gpu_coverage_job) self.assertNotIn("\n commit:", workflow) - self.assertIn('"sha=${COVERAGE_SHA}"', gpu_coverage_job) - self.assertNotIn('"sha=${GITHUB_SHA}"', gpu_coverage_job) client_readme = (ROOT / "client" / "README.md").read_text(encoding="utf-8") self.assertEqual(client_readme.count("flag=client-gpu"), 1) self.assertNotIn("flag=client)]", client_readme) - self.assertIn("for flag in client-unit client-gpu", gpu_coverage_job) - self.assertIn("for attempt in {1..90}", gpu_coverage_job) - self.assertIn("(${attempt}/90)", gpu_coverage_job) - self.assertIn("within 15 minutes", gpu_coverage_job) - self.assertNotIn("for attempt in {1..30}", gpu_coverage_job) - self.assertIn('"flag=${flag}"', gpu_coverage_job) - self.assertIn("'.totals.sessions // 0'", gpu_coverage_job) - self.assertIn('[ "${sessions}" -lt 1 ]', gpu_coverage_job) - self.assertIn('[ "${reports_ready}" = true ]', gpu_coverage_job) aggregate = workflow[workflow.index(" classic-validation:") :] self.assertIn("- gpu-coverage", aggregate) self.assertIn("--gpu-coverage-required", aggregate) - self.assertIn("--gpu-coverage-result '${{ needs.gpu-coverage.result }}'", aggregate) + self.assertIn( + "--gpu-coverage-result '${{ needs.gpu-coverage.result }}'", aggregate + ) expected_materials = { "core": { ".github/workflows/check.yml", @@ -1989,75 +1979,49 @@ def test_linux_checks_pin_image_and_isolate_compiler_caches(self) -> None: self.assertIn(f"printf '{field}\\t", measurement) - def run_gpu_codecov_poll(self, mode: str, tls_code: int = 60): + def test_codecov_reporting_is_informational_without_weakening_native_coverage( + self, + ) -> None: + config = (ROOT / "codecov.yml").read_text(encoding="utf-8") + self.assertIn(" project:\n default:\n informational: true", config) + self.assertIn(" patch:\n default:\n informational: true", config) + workflow = self.text("check.yml") - step = workflow.split(" - name: Wait for complete client coverage processing\n", 1)[1] - script = textwrap.dedent(step.split(" run: |\n", 1)[1].split("\n integrated:", 1)[0]) - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - curl = root / "curl" - curl.write_text(textwrap.dedent('''\ - #!/usr/bin/env python3 - import json, os, pathlib, sys - calls = pathlib.Path(os.environ['MOCK_CALLS']) - previous = calls.read_text().splitlines() if calls.exists() else [] - with calls.open('a') as stream: stream.write(json.dumps(sys.argv[1:]) + '\\n') - mode = os.environ['MOCK_MODE'] - if mode == 'tls': sys.exit(int(os.environ['MOCK_TLS_CODE'])) - if mode == 'http' and not previous: sys.exit(22) - sessions = 0 if mode == 'zero' or (mode == 'pending' and not previous) else 1 - print(json.dumps({'totals': {'sessions': sessions}})) - ''')) - jq = root / "jq" - jq.write_text(textwrap.dedent('''\ - #!/usr/bin/env python3 - import json, sys - print(json.load(sys.stdin)['totals']['sessions']) - ''')) - sleep = root / "sleep" - sleep.write_text('#!/bin/sh\nprintf "sleep\\n" >> "$MOCK_SLEEPS"\n') - for executable in (curl, jq, sleep): executable.chmod(0o700) - environment = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH'], - MOCK_CALLS=str(root / 'calls'), MOCK_SLEEPS=str(root / 'sleeps'), - MOCK_MODE=mode, MOCK_TLS_CODE=str(tls_code), - COVERAGE_SHA='a' * 40, - CODECOV_REPORT_API='https://api.codecov.io/api/v2/github/atrinik/repos/classic/totals/') - result = subprocess.run(['bash', '-c', script], env=environment, - capture_output=True, text=True, timeout=30) - calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] - sleeps = (root / 'sleeps').read_text().splitlines() if (root / 'sleeps').exists() else [] - return result, calls, sleeps - - def test_gpu_codecov_permanent_tls_errors_fail_without_repolling(self) -> None: - for code in (35, 51, 58, 60, 77): - with self.subTest(code=code): - result, calls, sleeps = self.run_gpu_codecov_poll('tls', code) - self.assertEqual(result.returncode, code, result.stderr) - self.assertIn(f'Codecov TLS transfer failed (curl exit {code})', result.stderr) - self.assertEqual(len(calls), 1) - self.assertEqual(sleeps, []) - self.assertNotIn('--retry', calls[0]) - self.assertNotIn('--insecure', calls[0]) - - def test_gpu_codecov_processing_and_http_delays_still_wait_for_both_reports(self) -> None: - for mode in ('pending', 'http'): - with self.subTest(mode=mode): - result, calls, sleeps = self.run_gpu_codecov_poll(mode) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(len(calls), 3) - self.assertEqual(sleeps, ['sleep']) - self.assertIn('flag=client-unit', calls[1]) - self.assertIn('flag=client-gpu', calls[2]) - for call in calls: - self.assertIn('sha=' + 'a' * 40, call) - self.assertEqual(call[-1], 'https://api.codecov.io/api/v2/github/atrinik/repos/classic/totals/') - - def test_gpu_codecov_zero_sessions_remains_a_required_timeout_failure(self) -> None: - result, calls, sleeps = self.run_gpu_codecov_poll('zero') - self.assertEqual(result.returncode, 1) - self.assertEqual(len(calls), 90) - self.assertEqual(len(sleeps), 90) - self.assertIn('within 15 minutes', result.stderr) + self.assertEqual(workflow.count("uses: codecov/codecov-action@"), 5) + self.assertEqual(workflow.count("fail_ci_if_error: false"), 5) + self.assertNotIn("fail_ci_if_error: true", workflow) + self.assertNotIn("api.codecov.io", workflow) + upload_steps = [ + re.split(r"(?m)^ [a-zA-Z0-9_-]+:", step, maxsplit=1)[0] + for step in re.split(r"(?m)(?=^ - )", workflow) + if "uses: codecov/codecov-action@" in step + ] + self.assertEqual(len(upload_steps), 5) + for step in upload_steps: + with self.subTest(upload_step=step.splitlines()[0]): + self.assertIn(" continue-on-error: true", step) + self.assertIn(" timeout-minutes: 2", step) + self.assertIn(" fail_ci_if_error: false", step) + + gpu_coverage = workflow[ + workflow.index(" gpu-coverage:\n name: Trusted GPU renderer coverage") : + workflow.index(" integrated:\n name: Integrated client/server graph") + ] + self.assertIn("tools/ci/run_gpu_coverage.sh", gpu_coverage) + self.assertIn("files: client/coverage.xml", gpu_coverage) + self.assertIn("flags: client-gpu", gpu_coverage) + native_coverage = gpu_coverage.split( + " - name: Upload GPU-complete client coverage", 1 + )[0] + self.assertIn("name: Build and exercise complete GPU coverage", native_coverage) + self.assertNotIn("continue-on-error: true", native_coverage) + + aggregate = workflow[workflow.index(" classic-validation:") :] + self.assertIn("- gpu-coverage", aggregate) + self.assertIn("--gpu-coverage-required", aggregate) + self.assertIn( + "--gpu-coverage-result '${{ needs.gpu-coverage.result }}'", aggregate + ) From 571a6570b677f3e534ec8fd507ab508c1b1c2774 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 5 Oct 2026 20:57:50 -0500 Subject: [PATCH 2/2] ci: classify Codecov config as shared validation --- tools/ci/classify_changes.py | 1 + tools/tests/test_classify_changes.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tools/ci/classify_changes.py b/tools/ci/classify_changes.py index 2fd48bf34..7f92e17a9 100755 --- a/tools/ci/classify_changes.py +++ b/tools/ci/classify_changes.py @@ -21,6 +21,7 @@ "ATTRIBUTIONS.md", "CMakeLists.txt", "CMakePresets.json", + "codecov.yml", "LICENSE.md", "docs/history/component-release-map.json", "docs/history/release-tags.json", diff --git a/tools/tests/test_classify_changes.py b/tools/tests/test_classify_changes.py index 2812bb1fb..a61daac73 100644 --- a/tools/tests/test_classify_changes.py +++ b/tools/tests/test_classify_changes.py @@ -79,6 +79,11 @@ def test_shared_native_change_selects_both_components(self) -> None: self.assertFalse(result["codeql_server"]) self.assertTrue(result["codeql_core_cpp"]) + def test_codecov_configuration_selects_both_coverage_components(self) -> None: + result = classify_changes.classify(["codecov.yml"]) + self.assertTrue(result["client"]) + self.assertTrue(result["server"]) + def test_release_configuration_selects_both_native_components(self) -> None: result = classify_changes.classify([".releaserc.cjs"]) self.assertTrue(result["client"])