From 0bd142535011fc2eae180d1a1a493c33702ace5d Mon Sep 17 00:00:00 2001 From: Ryan Bliss Date: Tue, 8 Sep 2026 15:25:40 -0700 Subject: [PATCH] Wait out registry propagation before deciding a version is unpublished The release step reads npm right after the publish step, and npm serves a publish it just accepted a few seconds later, so the last packages published can still be missing. It now re-reads only the missing versions on a bounded backoff (60s total) before failing, and says how long it waited. --- .github/workflows/release.yml | 7 ++- CONTRIBUTING.md | 5 +- scripts/create-github-release.mjs | 39 ++++++++---- scripts/publish-package-selection.mjs | 46 ++++++++++++++ scripts/publish-package-selection.spec.mjs | 73 ++++++++++++++++++++++ 5 files changed, 155 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8571da..07fe5b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,9 +96,10 @@ jobs: # Tag and release run even when a publish step failed or was # skipped: the script checks npm itself and refuses to tag until # every package in the release is published at its manifest - # version. So a partial publish leaves no tag, and the next run - # after the missing packages land (rerun or manual publish) - # creates the release. An existing release is left alone. + # version, waiting out registry propagation before it decides a + # version is missing. So a partial publish leaves no tag, and the + # next run after the missing packages land (rerun or manual + # publish) creates the release. An existing release is left alone. - name: Tag and create GitHub release (lockstep family) if: ${{ !cancelled() && steps.version.outcome == 'success' }} env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e04d81e..e401523 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,7 +115,10 @@ notes. The lockstep family shares one tag (`v0.8.0`); the React ESLint plugin gets a name-scoped one (`react-eslint-plugin-v0.1.1`). A failed package does not stop the packages after it from publishing. The tag step runs regardless and checks npm itself: it refuses to tag until every package in the release is -published, and leaves an existing release alone. So after a partial failure, +published, and leaves an existing release alone. The registry needs a few +seconds to serve a publish it just accepted, so a missing version is re-read on +a bounded backoff (about a minute) before it counts as unpublished — the error +says how long it waited. So after a partial failure, publish the missing packages (a rerun, or by hand) and rerun the workflow to create the release. Preview the notes for the current manifests with: diff --git a/scripts/create-github-release.mjs b/scripts/create-github-release.mjs index 37ca87d..65d7ada 100644 --- a/scripts/create-github-release.mjs +++ b/scripts/create-github-release.mjs @@ -7,10 +7,12 @@ * Runs at the end of the `Release` workflow whether or not the publish step * succeeded, and checks the registry itself: every package in the release must * already be on npm at its manifest version, so a tag never points at a - * version that failed to publish. Idempotent in the same way as the publish - * step: an existing release for the tag is left alone. So a partial publish - * leaves no tag, and the next run after the missing packages are published - * (by a rerun or by hand) creates it. + * version that failed to publish. The registry serves a publish it just + * accepted a few seconds later, so a version still missing is polled on a + * bounded backoff before it counts as unpublished. Idempotent in the same way + * as the publish step: an existing release for the tag is left alone. So a + * partial publish leaves no tag, and the next run after the missing packages + * are published (by a rerun or by hand) creates it. * * Two release trains, two tag shapes: * @@ -27,8 +29,9 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { - npmVersionState, + findUnpublishedVersions, publishPackageCatalog, + registryPropagationDelaysMs, } from "./publish-package-selection.mjs"; const rootDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -52,16 +55,30 @@ if (releaseExists(target.tag)) { process.exit(0); } -const unpublished = target.packages.filter( - (entry) => !npmVersionState(entry.label, entry.version).versionPublished -); -if (unpublished.length > 0) { - const list = unpublished +// A dry run answers from the registry as it reads right now; only the real +// run waits out propagation. +const { missing, waitedMs } = await findUnpublishedVersions(target.packages, { + delaysMs: options.dryRun ? [] : registryPropagationDelaysMs, + onWait: ({ missing: pending, delayMs }) => { + const list = pending.map((entry) => entry.label).join(", "); + console.log( + `Not on npm yet: ${list}. Waiting ${ + delayMs / 1000 + }s for the registry to catch up.` + ); + }, +}); +if (missing.length > 0) { + const list = missing .map((entry) => `${entry.label}@${entry.version}`) .join(", "); if (!options.dryRun) { throw new Error( - `create-github-release: ${list} is not on npm, so ${target.tag} would point at an unpublished version. Fix: publish the missing package(s) (\`npm run publish:packages\` skips the ones already on npm), then rerun the Release workflow to create the release.` + `create-github-release: ${list} is not on npm after waiting ${ + waitedMs / 1000 + }s for the registry, so ${ + target.tag + } would point at an unpublished version. Fix: publish the missing package(s) (\`npm run publish:packages\` skips the ones already on npm), then rerun the Release workflow to create the release.` ); } console.log(`Not on npm yet: ${list}. The real run would stop here.`); diff --git a/scripts/publish-package-selection.mjs b/scripts/publish-package-selection.mjs index 0b935cd..9fe9713 100644 --- a/scripts/publish-package-selection.mjs +++ b/scripts/publish-package-selection.mjs @@ -203,6 +203,52 @@ export function npmVersionState(packageName, version, env = process.env) { ); } +/** + * How long to wait between registry reads while a just-published version + * propagates. npm accepts a publish before every read replica serves it, so + * the packages published last are the ones a read right after the publish + * step can still miss. + */ +export const registryPropagationDelaysMs = [2000, 4000, 8000, 16000, 30000]; + +/** + * Which of `packages` ({ label, version }) are still not on npm, after giving + * the registry `delaysMs` chances to catch up. Re-reads only the versions + * still missing, and returns how long it waited so a caller can say so. + */ +export async function findUnpublishedVersions( + packages, + { + readVersionState = npmVersionState, + delaysMs = registryPropagationDelaysMs, + wait = sleep, + onWait = () => {}, + } = {} +) { + let missing = packages.filter( + (entry) => + !readVersionState(entry.label, entry.version).versionPublished + ); + let waitedMs = 0; + for (const delayMs of delaysMs) { + if (missing.length === 0) { + return { missing, waitedMs }; + } + onWait({ missing, delayMs }); + await wait(delayMs); + waitedMs += delayMs; + missing = missing.filter( + (entry) => + !readVersionState(entry.label, entry.version).versionPublished + ); + } + return { missing, waitedMs }; +} + +function sleep(delayMs) { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + function isNpmNotFound(stderr) { if (stderr.includes("E404")) { return true; diff --git a/scripts/publish-package-selection.spec.mjs b/scripts/publish-package-selection.spec.mjs index 15dea4c..62c701c 100644 --- a/scripts/publish-package-selection.spec.mjs +++ b/scripts/publish-package-selection.spec.mjs @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { buildNpmPublishArguments, + findUnpublishedVersions, parsePublishArguments, selectPackagesToPublish, } from "./publish-package-selection.mjs"; @@ -113,3 +114,75 @@ describe("publish package selection", () => { ).toEqual(["publish", "--provenance"]); }); }); + +describe("registry propagation", () => { + const family = [ + { label: "@retreejs/core", version: "0.11.1" }, + { label: "@retreejs/create", version: "0.11.1" }, + ]; + + /** Reports a package as published once `reads` reads have seen it. */ + function readerAfter(reads) { + const seen = new Map(); + return (label) => { + const count = (seen.get(label) ?? 0) + 1; + seen.set(label, count); + return { + packageExists: true, + versionPublished: count > reads[label], + }; + }; + } + + it("reads once and waits none when every version is already visible", async () => { + const waits = []; + + const result = await findUnpublishedVersions(family, { + readVersionState: readerAfter({ + "@retreejs/core": 0, + "@retreejs/create": 0, + }), + wait: (delayMs) => waits.push(delayMs), + }); + + expect(result).toEqual({ missing: [], waitedMs: 0 }); + expect(waits).toEqual([]); + }); + + it("waits out a version the registry has not served yet", async () => { + const waits = []; + const pending = []; + + const result = await findUnpublishedVersions(family, { + readVersionState: readerAfter({ + "@retreejs/core": 0, + "@retreejs/create": 2, + }), + delaysMs: [2000, 4000, 8000], + wait: (delayMs) => waits.push(delayMs), + onWait: ({ missing }) => + pending.push(missing.map((entry) => entry.label)), + }); + + expect(result).toEqual({ missing: [], waitedMs: 6000 }); + expect(waits).toEqual([2000, 4000]); + // Only the still-missing package is re-read. + expect(pending).toEqual([["@retreejs/create"], ["@retreejs/create"]]); + }); + + it("reports what is still missing and how long it waited", async () => { + const result = await findUnpublishedVersions(family, { + readVersionState: readerAfter({ + "@retreejs/core": 0, + "@retreejs/create": Number.POSITIVE_INFINITY, + }), + delaysMs: [2000, 4000], + wait: () => {}, + }); + + expect(result.missing).toEqual([ + { label: "@retreejs/create", version: "0.11.1" }, + ]); + expect(result.waitedMs).toBe(6000); + }); +});