Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions .github/workflows/base-std-docs-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -787,10 +787,16 @@ jobs:
# from client_payload — and let the script skip them for routing.
# A rename counts its previous name as removed.
#
# Best-effort: an API failure leaves removed_paths empty and logs a
# A rename with zero content changes (status `renamed`, `changes == 0`)
# also lists its new name under `unchanged_renames`: a pure move
# carries nothing to sync, so the script skips it for routing. Without
# this, a folder move such as base-std#240 (docs/ -> docs/b20/) would
# hand the model a header-only diff for every B20 page.
#
# Best-effort: an API failure leaves both lists empty and logs a
# warning rather than failing a sync whose content is otherwise fine.
# Runs for every non-release dispatch so the payload's own
# removed_paths, if any, is always overwritten.
# removed_paths / unchanged_renames, if any, are always overwritten.
if: env.PAYLOAD_KIND != 'release'
env:
SOURCE_REPO: ${{ env.PAYLOAD_SOURCE_REPO }}
Expand All @@ -803,9 +809,9 @@ jobs:
set -euo pipefail

if [[ -z "${SHA:-}" ]]; then
jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
jq '.removed_paths = [] | .unchanged_renames = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
echo "No sha in payload; removed_paths cleared."
echo "No sha in payload; removed_paths and unchanged_renames cleared."
exit 0
fi

Expand All @@ -817,23 +823,31 @@ jobs:
"https://api.github.com/repos/${SOURCE_REPO}/commits/${SHA}") || code="000"
if [[ "$code" != "200" ]]; then
echo "::warning title=Removed paths unavailable::reading commit ${SHA:0:7} on ${SOURCE_REPO} failed (HTTP ${code}); deletions will route like edits this run"
jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
jq '.removed_paths = [] | .unchanged_renames = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
exit 0
fi
jq -e '(.files // []) | all(.[]; (.filename | type == "string"))' "$commit" >/dev/null \
|| { echo "::warning title=Removed paths unavailable::commit ${SHA:0:7} returned malformed file metadata"; jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"; mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"; exit 0; }
|| { echo "::warning title=Removed paths unavailable::commit ${SHA:0:7} returned malformed file metadata"; jq '.removed_paths = [] | .unchanged_renames = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"; mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"; exit 0; }
jq --argjson cap "$MAX_REMOVED_PATHS" --argjson bytes "$MAX_REMOVED_PATH_BYTES" '
[ .files[]?
| select(.status == "removed" or .status == "renamed")
| (if .status == "renamed" then .previous_filename else .filename end)
| select(type == "string" and length > 0 and length <= $bytes)
] | unique | .[0:$cap]
' "$commit" > "$RUNNER_TEMP/trusted-removed-paths.json"
jq --argjson cap "$MAX_REMOVED_PATHS" --argjson bytes "$MAX_REMOVED_PATH_BYTES" '
[ .files[]?
| select(.status == "renamed" and ((.changes // 1) == 0))
| .filename
| select(type == "string" and length > 0 and length <= $bytes)
] | unique | .[0:$cap]
' "$commit" > "$RUNNER_TEMP/trusted-unchanged-renames.json"
jq --slurpfile removed "$RUNNER_TEMP/trusted-removed-paths.json" \
'.removed_paths = $removed[0]' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
--slurpfile moved "$RUNNER_TEMP/trusted-unchanged-renames.json" \
'.removed_paths = $removed[0] | .unchanged_renames = $moved[0]' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
echo "::notice title=Trusted removed paths derived::sha=${SHA:0:7} removed_paths=$(jq length "$RUNNER_TEMP/trusted-removed-paths.json")"
echo "::notice title=Trusted removed paths derived::sha=${SHA:0:7} removed_paths=$(jq length "$RUNNER_TEMP/trusted-removed-paths.json") unchanged_renames=$(jq length "$RUNNER_TEMP/trusted-unchanged-renames.json")"

- name: Derive trusted release routing inputs
# client_payload.changed_paths is attacker-controlled JSON. For a
Expand Down
32 changes: 29 additions & 3 deletions scripts/sync-from-base-std/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ bucket, logged as `[routing]` and reported in the PR body (or an issue):
| ignored | matched only a `kind: "ignored"` rule | nothing; deliberately unsynced (upstream README, authoring templates) |
| unrouted | matched no rule | listed under **Unrouted source files** with a guideline-derived placement proposal |
| removed | deleted in the source commit (`removed_paths`) | never routed; listed under **Removed source files** |
| moved | renamed with no content change (`unchanged_renames`) | never routed; listed under **Moved source files** |

`removed_paths` is derived by the workflow from the commit API for the verified
sha, never from the dispatcher's payload. A deleted documentation file carries
Expand All @@ -38,6 +39,28 @@ has been removed" banners on reference pages (base/docs#1928); the validator now
rejects callouts that describe repository housekeeping (`validateCallouts` in
`safety.mjs`).

`unchanged_renames` comes from the same commit API call: the new name of every
file GitHub reports as `renamed` with zero changes. A pure move carries nothing
to sync, so a folder move like base-std#240 does not send every B20 page to the
model with a header-only diff. A rename that also edits content still routes by
its new path.

### Feature folders in base-std

base-std is the Base Standard Library: one `docs/<feature>/` folder per Base
standard (base-std#240). Only B20 is mapped today:

| base-std source | Base Docs target |
|---|---|
| `docs/b20/**` | the B20 specification under `docs/specifications/b20/` and the Build on Base task pages each rule names |
| `changelog/NN_<Hardfork>_<Product>_<feature>.md` where Product is `B20*`, `PolicyRegistry`, or `ActivationRegistry` | its own entry page under `docs/base-chain/specs/reference/b20/changelog/`, plus a row on the B20 changelog summary |
| any other `docs/<feature>/` folder or changelog product (e.g. BaseTime) | unrouted: listed in the PR or issue body until a rule maps it |

To onboard a new feature, add `product-doc` rules for its `docs/<feature>/`
files pointing at the Base Docs pages that own it, and widen the changelog
entry `source_pattern` (or add a second `changelog-entry` rule with its own
`page_template`).

### Placement proposals from the IA guidelines

Unrouted Markdown sources go through one Haiku call (`proposePlacement`) that
Expand Down Expand Up @@ -89,9 +112,12 @@ LLM_GATEWAY_API_KEY=... \
--payload scripts/sync-from-base-std/fixtures/code-change-ib20.json
```

`fixtures/code-change-docs-restructure.json` is the real file list from
base-std@be6d045 with its `removed_paths`; run it the same way to exercise the
docs-tree routes and the placement pass (set `RUNNER_TEMP` to see the routing
`fixtures/code-change-docs-b20-move.json` is the real file list from
base-std#240 (docs moved under `docs/b20/`) with its `removed_paths` and
`unchanged_renames`; the routing tests use it. `fixtures/code-change-docs-restructure.json`
is the older base-std@be6d045 file list (pre-#240 paths, so its docs files are
now unrouted); run either the same way to exercise the docs-tree routes and the
placement pass (set `RUNNER_TEMP` to see the routing
report written as `sync-review.md`).

Configuration knobs are optional positive numbers:
Expand Down
88 changes: 65 additions & 23 deletions scripts/sync-from-base-std/__tests__/base-std-routing.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ test("loadKnownRoutes: the central Interfaces page is reachable", async () => {
// the sync edited pages from an all-minus diff and dropped 15 new files
// silently. These tests pin the behavior that replaced that.

const RESTRUCTURE_FIXTURE = "scripts/sync-from-base-std/fixtures/code-change-docs-restructure.json";
const RESTRUCTURE_FIXTURE = "scripts/sync-from-base-std/fixtures/code-change-docs-b20-move.json";

async function loadRouteTable() {
return JSON.parse(
Expand All @@ -330,39 +330,55 @@ test("upstream docs tree routes to the pages the IA guidelines assign", async ()

// Architecture: chain-generic precompile mechanics → Base Protocol → Execution;
// B20 component map → spec overview; execution/versioning invariants → invariants page.
const arch = await pagesFor("docs/architecture.md");
const arch = await pagesFor("docs/b20/architecture.md");
for (const expected of [
"docs/specifications/base-protocol/execution/precompiles.mdx",
`${B20_REFERENCE_ROOT}/index.mdx`,
`${B20_REFERENCE_ROOT}/reference/interfaces.mdx`,
]) {
assert.ok(arch.includes(expected), `docs/architecture.md should route to ${expected}`);
assert.ok(arch.includes(expected), `docs/b20/architecture.md should route to ${expected}`);
}
assert.ok(!arch.some((p) => p.includes("/i-policy-registry/")), "architecture must not fan out to the policy registry subtree");

// Concepts feed the spec overview key-concept sections plus the owning reference pages.
const multipliers = await pagesFor("docs/concepts/multipliers.md");
const multipliers = await pagesFor("docs/b20/concepts/multipliers.md");
assert.ok(multipliers.includes(`${B20_REFERENCE_ROOT}/index.mdx`));
assert.ok(multipliers.includes(`${B20_REFERENCE_ROOT}/reference/interfaces.mdx`));
assert.ok(multipliers.includes("docs/build-on-base/issue-rwa/apply-a-multiplier.mdx"));

// Guides feed the existing Build on Base task pages (ia-guidelines: Tokenize Assets / Issue Stablecoins).
assert.ok((await pagesFor("docs/guides/scheduling-stock-splits.md")).includes("docs/build-on-base/issue-rwa/apply-a-multiplier.mdx"));
assert.ok((await pagesFor("docs/guides/announcing-corporate-actions.md")).includes("docs/build-on-base/issue-rwa/announce-a-distribution.mdx"));
assert.ok((await pagesFor("docs/guides/restricting-transfer-initiators.md")).includes("docs/build-on-base/issue-rwa/restrict-transfer-initiators.mdx"));
const seize = await pagesFor("docs/guides/seizeing-assets.md");
assert.ok((await pagesFor("docs/b20/guides/scheduling-stock-splits.md")).includes("docs/build-on-base/issue-rwa/apply-a-multiplier.mdx"));
assert.ok((await pagesFor("docs/b20/guides/announcing-corporate-actions.md")).includes("docs/build-on-base/issue-rwa/announce-a-distribution.mdx"));
assert.ok((await pagesFor("docs/b20/guides/restricting-transfer-initiators.md")).includes("docs/build-on-base/issue-rwa/restrict-transfer-initiators.mdx"));
const seize = await pagesFor("docs/b20/guides/seizeing-assets.md");
assert.ok(seize.includes("docs/build-on-base/issue-rwa/seize-and-cancel-units.mdx"));
assert.ok(seize.includes("docs/build-on-base/issue-stablecoins/recover-funds.mdx"));
assert.ok(seize.includes(`${B20_REFERENCE_ROOT}/reference/interfaces.mdx`));

// Reference tables feed the supporting pages.
assert.deepEqual(await pagesFor("docs/reference/constants.md"), [`${B20_REFERENCE_ROOT}/reference/constants.mdx`]);
assert.deepEqual(await pagesFor("docs/reference/errors.md"), [`${B20_REFERENCE_ROOT}/reference/errors.mdx`]);
assert.deepEqual(await pagesFor("docs/reference/events.md"), [`${B20_REFERENCE_ROOT}/reference/events.mdx`]);
assert.deepEqual(await pagesFor("docs/reference/interfaces.md"), [`${B20_REFERENCE_ROOT}/reference/interfaces.mdx`]);

// Scaffolding is explicitly ignored, and the retired flat tree no longer routes anywhere.
for (const src of ["docs/guides/template.md", "docs/README.md", "README.md", "docs/B20/Asset.md", "docs/PolicyRegistry/README.md"]) {
assert.deepEqual(await pagesFor("docs/b20/reference/constants.md"), [`${B20_REFERENCE_ROOT}/reference/constants.mdx`]);
assert.deepEqual(await pagesFor("docs/b20/reference/errors.md"), [`${B20_REFERENCE_ROOT}/reference/errors.mdx`, `${B20_REFERENCE_ROOT}/reference/events.mdx`]);
assert.deepEqual(await pagesFor("docs/b20/reference/events.md"), [`${B20_REFERENCE_ROOT}/reference/errors.mdx`, `${B20_REFERENCE_ROOT}/reference/events.mdx`]);
assert.deepEqual((await pagesFor("docs/b20/integration-architecture.md")).sort(), [
`${B20_REFERENCE_ROOT}/index.mdx`,
"docs/specifications/base-protocol/execution/precompiles.mdx",
]);
assert.deepEqual(await pagesFor("docs/b20/reference/interfaces.md"), [`${B20_REFERENCE_ROOT}/reference/interfaces.mdx`]);

// Scaffolding is explicitly ignored; the retired flat trees (pre-#213 docs/B20/, pre-#240 docs/<topic>/)
// and other features' folders no longer route anywhere.
for (const src of [
"docs/b20/guides/template.md",
"docs/b20/README.md",
"docs/README.md",
"README.md",
"docs/B20/Asset.md",
"docs/PolicyRegistry/README.md",
"docs/overview.md",
"docs/concepts/policies.md",
"docs/reference/errors.md",
"docs/basetime/overview.md",
]) {
assert.deepEqual(await pagesFor(src), [], `${src} must not route`);
}
});
Expand All @@ -383,21 +399,47 @@ test("removed source files never route, even when a rule still matches them", as
assert.deepEqual(work[0].sourceFiles, ["src/interfaces/IB20Asset.sol"]);
});

test("classifyChangedPaths separates routed, ignored, unrouted, and removed for the be6d045 fixture", async () => {
test("classifyChangedPaths separates routed, ignored, unrouted, removed, and moved for the base-std#240 fixture", async () => {
const routeTable = await loadRouteTable();
const fixture = JSON.parse(await fs.readFile(path.join(REPO_ROOT, RESTRUCTURE_FIXTURE), "utf8"));
const c = classifyChangedPaths(routeTable, fixture.changed_paths, { removedPaths: fixture.removed_paths });
const opts = { removedPaths: fixture.removed_paths, unchangedRenames: fixture.unchanged_renames };
const c = classifyChangedPaths(routeTable, fixture.changed_paths, opts);
assert.deepEqual(c.removed, fixture.removed_paths);
assert.deepEqual(c.ignored.sort(), ["README.md", "docs/README.md", "docs/guides/template.md"]);
assert.deepEqual(c.unrouted, [], "every surviving file in the restructure has a rule");
assert.equal(c.routed.length, fixture.changed_paths.length - c.removed.length - c.ignored.length);
assert.deepEqual(c.moved, fixture.unchanged_renames);
assert.deepEqual(c.ignored.sort(), ["AGENTS.md", "README.md", "docs/README.md", "docs/b20/README.md"]);
assert.deepEqual(c.unrouted, [], "every surviving file in the move has a rule");
// Every renamed doc that changed content still routes by its new docs/b20/ path.
for (const f of ["docs/b20/overview.md", "docs/b20/concepts/token-types.md", "docs/b20/reference/errors.md", "docs/b20/reference/interfaces.md"]) {
assert.ok(c.routed.includes(f), `${f} should route`);
}
// Pure moves produce no work; only files whose content changed are routed.
const work = await routeCodeChange(routeTable, fixture.changed_paths, { repoRoot: REPO_ROOT, ...opts });
const sources = new Set(work.flatMap((w) => w.sourceFiles));
for (const f of fixture.unchanged_renames) assert.ok(!sources.has(f), `${f} is a pure move and must not route`);
for (const f of fixture.removed_paths) assert.ok(!sources.has(f), `${f} was removed and must not route`);
assert.ok(sources.has("docs/b20/reference/errors.md"));
// A file outside every rule is reported, not dropped.
const c2 = classifyChangedPaths(routeTable, ["docs/concepts/brand-new-topic.md", "foundry.toml"]);
assert.deepEqual(c2.unrouted, ["docs/concepts/brand-new-topic.md", "foundry.toml"]);
assert.ok(isDocSource("docs/concepts/brand-new-topic.md"));
const c2 = classifyChangedPaths(routeTable, ["docs/b20/concepts/brand-new-topic.md", "foundry.toml"]);
assert.deepEqual(c2.unrouted, ["docs/b20/concepts/brand-new-topic.md", "foundry.toml"]);
assert.ok(isDocSource("docs/b20/concepts/brand-new-topic.md"));
assert.ok(!isDocSource("foundry.toml"));
});

test("changelog entries route to the B20 changelog only for B20 products", async () => {
const routeTable = await loadRouteTable();
const pagesFor = async (src) =>
(await routeCodeChange(routeTable, [src], { repoRoot: REPO_ROOT })).map((w) => w.page);
const entryDir = "docs/base-chain/specs/reference/b20/changelog";
assert.deepEqual(await pagesFor("changelog/03_Denim_B20_token_receiver.md"), [`${entryDir}/03-denim-b20-token-receiver.mdx`]);
assert.deepEqual(await pagesFor("changelog/02_Cobalt_B20Asset_multiplier.md"), [`${entryDir}/02-cobalt-b20asset-multiplier.mdx`]);
assert.deepEqual(await pagesFor("changelog/03_Denim_PolicyRegistry_not_policy.md"), [`${entryDir}/03-denim-policyregistry-not-policy.mdx`]);
assert.deepEqual(await pagesFor("changelog/04_Ember_ActivationRegistry_example.md"), [`${entryDir}/04-ember-activationregistry-example.mdx`]);
// Other Base standards in base-std are reported as unrouted, never filed under B20.
const baseTime = "changelog/03_Denim_BaseTime_millisecond_timestamp.md";
assert.deepEqual(await pagesFor(baseTime), []);
assert.deepEqual(classifyChangedPaths(routeTable, [baseTime]).unrouted, [baseTime]);
});

test("filterPlacementProposals keeps only real sources and existing candidate pages", () => {
const sources = ["docs/concepts/brand-new-topic.md"];
const candidates = [`${B20_REFERENCE_ROOT}/index.mdx`];
Expand Down
Loading
Loading