diff --git a/.changeset/docs-restructure-require-resolve.md b/.changeset/docs-restructure-require-resolve.md new file mode 100644 index 00000000..398cd798 --- /dev/null +++ b/.changeset/docs-restructure-require-resolve.md @@ -0,0 +1,16 @@ +--- +"@aws-blocks/blocks": patch +"@aws-blocks/create-blocks-app": patch +--- + +docs: per-block docs folders + committed BB catalog with CI sync check; CLAUDE/agents docs resolved via require.resolve + +`@aws-blocks/blocks` now ships one docs folder per Building Block under `docs//` +(`README.md` / `API.md` / `DESIGN.md`), plus a committed, marker-delimited Building Block +catalog in the package README that a `sync-docs --check` CI gate keeps in sync. The README's +catalog section and the scaffolded `AGENTS.md` (`@aws-blocks/create-blocks-app`) now direct +tools and agents to locate docs programmatically via +`require.resolve('@aws-blocks/blocks/docs//README.md')` (and +`require.resolve('@aws-blocks/blocks/docs/README.md')` for the catalog) rather than assuming a +`node_modules/` path or following the human-facing relative links. Also adds a Security +Considerations section to the package README. diff --git a/.github/workflows/block-catalog-check.yml b/.github/workflows/block-catalog-check.yml new file mode 100644 index 00000000..9127f0ef --- /dev/null +++ b/.github/workflows/block-catalog-check.yml @@ -0,0 +1,25 @@ +name: Block Catalog Check + +# Fails the PR if the generated Building Block catalog table in +# packages/blocks/README.md is out of date. The script only reads the committed +# package READMEs + root README (Node builtins, zero deps), so no `npm ci` needed. +# On failure its stderr tells the engineer to run `npm run sync-docs` and commit. + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + check: + name: Catalog in sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version-file: '.nvmrc' + - name: Verify Building Block catalog is in sync + run: node scripts/sync-catalog.mjs --check diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index e7cc069f..b9278a2e 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -72,7 +72,7 @@ jobs: - run: npm ci - name: Sync block docs (prebuild step) - run: node scripts/sync-block-docs.mjs + run: node scripts/gen-block-docs.mjs - name: Check blocks docs-sync integrity env: @@ -86,14 +86,16 @@ jobs: exit 0 fi - # sync-block-docs.mjs regenerates packages/blocks/docs/ from sibling READMEs + # gen-block-docs.mjs regenerates packages/blocks/docs/ from sibling READMEs # (docs/ is gitignored, built at publish time). We compare the packed # @aws-blocks/blocks tarball at THIS PR's head against the PR base branch — # like-for-like (same algo, same build state), so only a change introduced by # THIS PR trips the check. A PR that doesn't alter what ships needs no bump. + # The base pack falls back to the pre-rename sync-block-docs.mjs so this job works + # against bases from either side of the gen-block-docs.mjs rename. pack_blocks() { local dest="$1" - node scripts/sync-block-docs.mjs >/dev/null + node scripts/gen-block-docs.mjs >/dev/null # --ignore-scripts intentionally packs source + synced docs, NOT a freshly built dist/ — this diffs shipped content, not the published build. ( cd packages/blocks && npm pack --ignore-scripts --pack-destination "$dest" >/dev/null ) set -- "$dest"/*.tgz @@ -109,7 +111,7 @@ jobs: mkdir -p /tmp/base-pack BASE_INTEGRITY=$( cd /tmp/blocks-base - node scripts/sync-block-docs.mjs >/dev/null + node scripts/gen-block-docs.mjs >/dev/null 2>/dev/null || node scripts/sync-block-docs.mjs >/dev/null ( cd packages/blocks && npm pack --ignore-scripts --pack-destination /tmp/base-pack >/dev/null ) set -- /tmp/base-pack/*.tgz [ $# -eq 1 ] || { echo "expected exactly one .tgz in /tmp/base-pack, found $#" >&2; exit 1; } diff --git a/package.json b/package.json index 44d63baa..9120999e 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,8 @@ "create-demo": "node packages/create-blocks-app/dist/index.js --template demo", "update-template": "tsx scripts/update-template-from-demo.ts", "build:tech-design-pdf": "node scripts/build-tech-design-pdf.mjs", + "sync-docs": "node scripts/sync-catalog.mjs --write", + "sync-docs:check": "node scripts/sync-catalog.mjs --check", "review-pr": "bash scripts/review-pr.sh", "publish:local": "tsx scripts/publish/publish.ts", "version": "changeset version && npm install --package-lock-only", diff --git a/packages/blocks/README.md b/packages/blocks/README.md index 67093de3..c3000b34 100644 --- a/packages/blocks/README.md +++ b/packages/blocks/README.md @@ -109,31 +109,78 @@ onAuthChange(authApi, (user) => { ## Building Blocks -Each block is its own package; full per-block docs ship in this package under **`docs/.md`**, and **`docs/index.md`** has a decision tree to help you pick. +Start from what you need: + +- **Store data** + - Simple key → value (caches, flags, user prefs) → `KVStore` (bb-kv-store) + - Structured records with indexes and queries → `DistributedTable` (bb-distributed-table) — **default for most data** + - Relational / SQL (joins, transactions) → see [Choosing a data block](#choosing-a-data-block) below + - Files, blobs, uploads, static assets → `FileBucket` (bb-file-bucket) + - A single config value or secret → `AppSetting` (bb-app-setting) +- **Authenticate users** + - Username/password, prototypes/MVPs → `AuthBasic` (bb-auth-basic) + - Cognito user pools, MFA, groups → `AuthCognito` (bb-auth-cognito) + - External identity provider (OIDC) → `AuthOIDC` (bb-auth-oidc) +- **Run work outside the request/response** + - Fire-and-forget background jobs → `AsyncJob` (bb-async-job) + - Scheduled / recurring tasks → `CronJob` (bb-cron-job) +- **Push live updates to browsers** (chat, presence, dashboards) → `Realtime` (bb-realtime) +- **Build AI features** + - Agent with tool use + conversation → `Agent` (bb-agent) + - Semantic document retrieval (RAG) → `KnowledgeBase` (bb-knowledge-base) +- **Send transactional email** → `EmailClient` (bb-email-client) +- **Serve a raw HTTP endpoint** (webhook receiver, health check, redirect, non-JSON response) → `RawRoute` (core); everything else goes through `ApiNamespace` RPC +- **Observe and operate** + - Structured logs → `Logger` (bb-logger) + - Custom metrics → `Metrics` (bb-metrics) + - Distributed traces → `Tracer` (bb-tracer) + - Auto CloudWatch dashboard → `Dashboard` (bb-dashboard) + +### Choosing a data block + +Default to `DistributedTable` for your data models unless your domain specifically requires SQL engine capabilities. + +Reach for one of the SQL blocks when you need to filter or join results across more than one related record, filter models on many dimensions with no preset hierarchy, store large objects, require transactions, or otherwise need the flexibility or familiarity of SQL that NoSQL does not offer. + +If you need SQL, prefer `DistributedDatabase` for basic Postgres-compatible querying. Use `Database` specifically when you need a full (more expensive) Postgres implementation where the engine itself provides and enforces foreign keys, row level security, triggers, views, large transactions (more than 3,000 rows), or integration with an existing Postgres database. Note it carries an idle cost at minimum 0.5 ACU, or a cold start when scaling from zero, unlike the other two blocks. + +## Building Block documentation + +Every Building Block ships its docs inside the `@aws-blocks/blocks` package under `docs//`: each `docs//` folder contains that block's `README.md`, plus `API.md`, `DESIGN.md`, and `CHANGELOG.md` where present. To read them, locate the bundled folder: -| Building Block | Import | Use it for | -|---|---|---| -| `Scope` | `@aws-blocks/blocks` | Resource boundaries / grouping for your backend | -| `ApiNamespace` | `@aws-blocks/blocks` | Type-safe APIs wired to the frontend automatically | -| `KVStore` | `@aws-blocks/blocks` | Simple key-value get/put/delete (prefs, flags, caches) | -| `DistributedTable` | `@aws-blocks/blocks` | Structured data with indexes and queries — **default for most data** | -| `DistributedDatabase` | `@aws-blocks/blocks` | Serverless SQL (Aurora DSQL) — zero-ops, scales to zero | -| `Database` | `@aws-blocks/blocks` | Full PostgreSQL (Aurora) — FKs, RLS, triggers, or an existing DB | -| `AuthBasic` | `@aws-blocks/blocks` | Username/password auth for prototypes, internal tools, MVPs | -| `AuthCognito` | `@aws-blocks/blocks` | Cognito User Pools — MFA, groups, hosted identity | -| `AuthOIDC` | `@aws-blocks/blocks` | Sign-in gated by an external OIDC identity provider | -| `Realtime` | `@aws-blocks/blocks` | Push to browsers — chat, presence, live dashboards | -| `AsyncJob` | `@aws-blocks/blocks` | Fire-and-forget background work (emails, uploads, reports) | -| `CronJob` | `@aws-blocks/blocks` | Scheduled / recurring tasks | -| `FileBucket` | `@aws-blocks/blocks` | File storage — uploads, downloads, presigned URLs | -| `AppSetting` | `@aws-blocks/blocks` | A single config value or secret (flags, API keys) | -| `KnowledgeBase` | `@aws-blocks/blocks` | Semantic document retrieval / RAG (Bedrock + S3 Vectors) | -| `Agent` | `@aws-blocks/blocks` | AI agent — tool use, streaming, conversation persistence | -| `EmailClient` | `@aws-blocks/blocks` | Transactional email (SES) | -| `Logger` / `Metrics` / `Tracer` / `Dashboard` | `@aws-blocks/blocks` | Observability — structured logs, metrics, traces, CloudWatch dashboard | -| `Hosting` | `@aws-blocks/blocks` | Deploy a frontend (SPA / static / Next.js SSR) on CloudFront + S3 | - -> **Not sure which data block?** Start with `DistributedTable` (DynamoDB). Reach for SQL only when you need joins across records, many-dimensional filtering, large transactions, or an existing Postgres database — `DistributedDatabase` for serverless Postgres, `Database` for full Aurora Postgres (FKs, RLS, triggers; carries idle cost / cold starts the other two don't). The full rationale is in `docs/index.md`. +```bash +node -p "require('path').dirname(require.resolve('@aws-blocks/blocks/docs/README.md'))" +``` + +If resolution fails, fall back to `node_modules/@aws-blocks/blocks/docs`. That folder holds this guide (`README.md`), the framework's own `API.md`, `TROUBLESHOOTING.md`, and `CHANGELOG.md` (version history — read when troubleshooting), plus one subfolder per block; the catalog below lists every block. + + +| Block | What it does | Keywords | +|-------|--------------|----------| +| auth-common | Shared interfaces and UI components for all AWS Blocks auth Building Blocks. | — | +| bb-agent | AI agent with streaming, tool calling, and conversation persistence. | — | +| bb-app-setting | A single application configuration value backed by SSM Parameter Store. | — | +| bb-async-job | Background job processing backed by SQS and Lambda. | queue, job, background, async, worker, submit, batch, retry, status, transitions, SQS | +| bb-auth-basic | Simple username/password authentication with JWT sessions, password policy, and optional code-confirmed signup and password reset. | — | +| bb-auth-cognito | Authentication backed by Amazon Cognito User Pools. | — | +| bb-auth-oidc | OIDC sign-in gate for AWS Blocks applications. | — | +| bb-cron-job | Scheduled task execution backed by EventBridge Scheduler and Lambda. | cron, schedule, timer, periodic, recurring, rate, EventBridge, background, interval | +| bb-dashboard | Auto-generated CloudWatch Dashboard for application observability. | — | +| bb-data | Full PostgreSQL database — provisions Aurora Serverless v2 by default, or connects to an existing PostgreSQL database (Supabase, Neon, etc.) via `fromExisting()`. | — | +| bb-distributed-data | Serverless SQL database backed by Amazon Aurora DSQL. | — | +| bb-distributed-table | Structured data storage backed by DynamoDB with secondary indexes and rich query capabilities. | — | +| bb-email-client | Transactional email sending via Amazon SES. | — | +| bb-file-bucket | File storage backed by Amazon S3. | — | +| bb-knowledge-base | Semantic document retrieval backed by Amazon Bedrock Knowledge Bases. | — | +| bb-kv-store | Simple key-value storage backed by DynamoDB. | — | +| bb-logger | Structured logging with consistent JSON format, log levels, and contextual metadata. | — | +| bb-metrics | Custom application metrics backed by Amazon CloudWatch (via Embedded Metric Format). | — | +| bb-realtime | Real-time pub/sub messaging backed by API Gateway WebSocket + DynamoDB. | — | +| bb-tracer | Distributed tracing backed by AWS X-Ray. | — | +| core | Core primitives for building full-stack applications with the AWS Blocks. | — | +| hosting | Low-level CDK L3 constructs for deploying web applications on AWS | — | +| pipeline | CDK Pipelines-based CI/CD construct for AWS Blocks applications. | — | + ## Local development and deploying @@ -144,6 +191,8 @@ Each block is its own package; full per-block docs ship in this package under ** | Data | persists to `.bb-data/` (delete to reset) | lives in AWS | | Use for | rapid iteration, tests | pre-production validation against real services | +> **Deploying needs AWS credentials.** `npm run dev` is fully local (no creds). `npm run sandbox` and `npm run deploy` provision real AWS resources, so configure credentials first — e.g. `aws configure sso` + `aws sso login`, or `aws configure` (verify with `aws sts get-caller-identity`). Use **least-privilege** credentials scoped to the services your blocks deploy — not broad `Administrator` access. + `npm run deploy` does a full production deploy; `npm run sandbox:destroy` tears the sandbox down. The same backend code runs in all three — blocks switch implementations automatically. `npm run deploy` streams CloudFormation events to **stdout** as they happen, so `npm run deploy | tee deploy.log` shows live progress instead of going silent for minutes, and a deploy failure keeps its reason on **stderr** (that stays the place to grep for why a deploy failed). On POSIX the deploy also survives a stray reap: a single `SIGTERM`, or any `SIGHUP` — a closed terminal, a backgrounded `npm run deploy &` — logs a line and keeps streaming rather than abandoning a stack update CloudFormation is still applying. Press Ctrl-C, or send `SIGTERM` twice, to stop it. That signal resilience is POSIX-only: Windows has no process groups and no OS-delivered `SIGTERM`/`SIGHUP`, so there a kill on the process tree still ends the deploy. @@ -183,9 +232,22 @@ Run with `npm run test:e2e`. Write the test first, iterate against mocks until i - **`Database` when `DistributedTable` would do** — Aurora costs more and has cold starts; reach for SQL only when you need it. - **Curling REST-style paths** — there is no `GET /api/getData`. All calls are JSON-RPC to a single `POST /aws-blocks/api`; use the typed import instead. +## Security Considerations + +- Use `await auth.requireAuth(context)` in every method that shouldn't be public — ApiNamespace methods are **unauthenticated by default** +- Use `new AppSetting(scope, id, { secret: true })` for API keys and credentials — never hardcode or use `.env` files +- Always attach a schema to KVStore/AppSetting that accepts user data — the RPC layer validates structure but not business logic +- Do not add broad `*` IAM policies — each Building Block already grants least-privilege scoped to its own resources +- Never change `blockPublicAccess` on FileBucket — serve public files through CloudFront instead +- Configure `CORS_ALLOWED_ORIGINS` explicitly for production — avoid wildcards +- For cross-domain deployments, pass `crossDomain: true` to auth constructors (enables `SameSite=None; Secure; Partitioned`) +- Enable `monitoring: { enabled: true, snsTopicArn: '...' }` on Hosting for production alerts +- Add WAF and API Gateway throttling via CDK for public-facing apps — not included by default +- Logger provides serialization safety (circular refs, type coercion) but does NOT redact sensitive content — never pass raw credentials, tokens, or secrets to Logger methods; sanitize context objects before logging + ## Reference -- **Per-block documentation:** `docs/.md` (e.g. `docs/bb-distributed-table.md`); `docs/index.md` for the catalog + decision tree. +- **Per-block documentation:** `docs//README.md` (overview), plus `docs//API.md` (full API reference) and `docs//DESIGN.md` (architecture & rationale) where present — e.g. `docs/bb-distributed-table/README.md`. The catalog + decision tree live in `docs/README.md`. - **UI components** (`@aws-blocks/blocks/ui`): `Authenticator`, `AuthenticatedContent`, `AccountMenuBar`, `onAuthChange`, `broadcastAuthChange` — framework-agnostic, return DOM nodes. See the `@aws-blocks/auth-common` README. - **SSR** (`@aws-blocks/blocks/server`): `withAuth` forwards browser cookies to API calls during server rendering. See the `@aws-blocks/core` README. - **Wire protocol & debugging:** the client is JSON-RPC 2.0 over a single endpoint — you should never call it directly. For `curl`-level troubleshooting, see [TROUBLESHOOTING.md](./TROUBLESHOOTING.md). diff --git a/packages/blocks/package.json b/packages/blocks/package.json index 93b57a96..8be26c03 100644 --- a/packages/blocks/package.json +++ b/packages/blocks/package.json @@ -45,10 +45,12 @@ "./utils": { "types": "./dist/utils.d.ts", "default": "./dist/utils.js" - } + }, + "./docs/*": "./docs/*" }, "scripts": { - "prebuild": "node ../../scripts/sync-block-docs.mjs", + "prebuild": "node ../../scripts/gen-block-docs.mjs", + "prepack": "node ../../scripts/gen-block-docs.mjs", "build": "tsc --build", "test": "node --test dist/conditional-exports.test.js dist/vendorize-map.test.js" }, diff --git a/packages/create-blocks-app/resources/AGENTS.md b/packages/create-blocks-app/resources/AGENTS.md index 8e0275f3..1a72240d 100644 --- a/packages/create-blocks-app/resources/AGENTS.md +++ b/packages/create-blocks-app/resources/AGENTS.md @@ -5,9 +5,7 @@ - **Backend:** `aws-blocks/index.ts` — APIs, auth, data models - **Frontend:** `src/` — imports backend APIs via `import { api } from 'aws-blocks'` - **Tests:** `test/e2e.test.ts` — run with `npm run test:e2e` -- **Full guide:** `node_modules/@aws-blocks/blocks/README.md` — architecture, workflow, best practices, common mistakes -- **Block catalog + decision tree:** `node_modules/@aws-blocks/blocks/docs/index.md` -- **Per-block docs:** `node_modules/@aws-blocks/blocks/docs/.md` +- **AWS Blocks docs** ship inside the `@aws-blocks/blocks` package. Find the docs folder once: `node -p "require('path').dirname(require.resolve('@aws-blocks/blocks/docs/README.md'))"` (fallback: `node_modules/@aws-blocks/blocks/docs`). Read everything relative to it: `README.md` (dev guide + catalog + decision tree — start here), then `/README.md`, plus `/API.md` and `/DESIGN.md` where present. ## Workflow @@ -19,7 +17,7 @@ ## Rules - **Use Building Blocks** for all persistence and cloud abstractions — never local files, in-memory arrays, or local databases. -- **Read block docs** at `node_modules/@aws-blocks/blocks/docs/.md` before using a block. +- **Read block docs** before using a block — start with its `README.md`, then `API.md` / `DESIGN.md` where present — not every block has them, so a missing file is not an error (see the **AWS Blocks docs** bullet above for where the docs folder lives). - **The JSON-RPC transport is invisible** — do not construct RPC payloads manually. Import and call the typed API directly. ## Deploying (requires AWS credentials) diff --git a/scripts/gen-block-docs.mjs b/scripts/gen-block-docs.mjs new file mode 100644 index 00000000..b717ed02 --- /dev/null +++ b/scripts/gen-block-docs.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Generates the gitignored, shipped `docs/` artifact for @aws-blocks/blocks. Runs + * as the packages/blocks `prebuild` AND `prepack` hooks so the published package + * always carries fresh docs — even when packing without a preceding build — + * without dirtying any tracked file. + * + * It produces three things under packages/blocks/docs/: + * 1. Per-block folders docs// — mirror every root-level *.md of each + * included package, so block-specific docs (README.md, API.md, DESIGN.md, + * CHANGELOG.md, ...) ship automatically. + * 2. docs//docs/ — if an included package has its own `docs/` folder + * (code samples, mock data, any extension), its entire contents are + * mirrored there verbatim, namespaced under the package so it can never + * collide with the root *.md copied into docs//. + * 3. docs/ root — verbatim copies of every root-level *.md of the umbrella + * packages/blocks package (README.md, API.md, TROUBLESHOOTING.md, + * CHANGELOG.md). `blocks` stays in EXCLUDED so these land at the docs/ root + * only, with no redundant docs/blocks/ subfolder. + * + * This script NEVER modifies packages/blocks/README.md. The committed catalog + * table inside that README is managed separately by scripts/sync-catalog.mjs + * (`npm run sync-docs`); run that and commit before building if you added or + * removed a block. + * + * No flag is required — the default run generates docs/. `--docs-only` is accepted + * as a harmless alias for the same behavior. + * + * Inclusion rule: every package under packages/ that has a README.md and is not in + * EXCLUDED. + * + * NOTE: EXCLUDED + getPackages() are intentionally duplicated in this file and in + * sync-catalog.mjs so each script stays dependency-free and independently + * runnable (no shared module to resolve, no build step). They MUST agree on the + * block set — keep the two in sync when editing. If this pair grows further, + * extract a shared module instead. + */ + +import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, statSync, rmSync, cpSync } from 'node:fs'; +import { join, dirname, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const packagesDir = join(__dirname, '..', 'packages'); +const blocksDir = join(packagesDir, 'blocks'); +const outDir = join(blocksDir, 'docs'); + +const EXCLUDED = new Set(['blocks', 'data-common', 'foundations', 'create-blocks-app']); + +const packages = getPackages(); + +generatePerBlockDocs(); +copyMarkdown(blocksDir, outDir); + +console.log(`Generated ${packages.length} block docs → packages/blocks/docs/`); + +// ─── docs/ artifact ────────────────────────────────────────────────────────── + +function generatePerBlockDocs() { + // Clean and recreate so removed blocks/files don't linger. + rmSync(outDir, { recursive: true, force: true }); + mkdirSync(outDir, { recursive: true }); + + for (const pkg of packages) { + const pkgDir = join(packagesDir, pkg); + const blockOutDir = join(outDir, pkg); + mkdirSync(blockOutDir, { recursive: true }); + + copyMarkdown(pkgDir, blockOutDir); + + const pkgDocsDir = join(pkgDir, 'docs'); + if (existsSync(pkgDocsDir) && statSync(pkgDocsDir).isDirectory()) { + const resolvedPkgDocsDir = resolve(pkgDocsDir); + const resolvedOutDir = resolve(outDir); + const isSelfReferential = + resolvedPkgDocsDir === resolvedOutDir || resolvedPkgDocsDir.startsWith(resolvedOutDir + sep); + if (!isSelfReferential) { + cpSync(pkgDocsDir, join(blockOutDir, 'docs'), { recursive: true }); + } + } + } +} + +/** Mirrors every root-level *.md file of `srcDir` into `destDir` verbatim. */ +function copyMarkdown(srcDir, destDir) { + const mdFiles = readdirSync(srcDir, { withFileTypes: true }).filter( + (entry) => entry.isFile() && entry.name.endsWith('.md'), + ); + for (const entry of mdFiles) { + writeFileSync(join(destDir, entry.name), readFileSync(join(srcDir, entry.name), 'utf-8')); + } +} + +// ─── Package discovery (duplicated from sync-catalog.mjs) ────────────────────── + +function getPackages() { + return readdirSync(packagesDir).filter( + (name) => !name.startsWith('.') && !EXCLUDED.has(name) && existsSync(join(packagesDir, name, 'README.md')), + ); +} diff --git a/scripts/publish/publish.ts b/scripts/publish/publish.ts index 5b3b8cc9..bebfeb4a 100644 --- a/scripts/publish/publish.ts +++ b/scripts/publish/publish.ts @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; -import { readdir, readFile, unlink, writeFile, mkdir, stat, access } from "node:fs/promises"; +import { readdir, readFile, unlink, writeFile, mkdir, stat, access, mkdtemp } from "node:fs/promises"; +import { readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join, resolve, relative, } from "node:path"; import { execSync } from "node:child_process"; import { LocalPublisher } from "./publishers/local.ts"; @@ -97,18 +99,28 @@ function topoSort(packages: PackageInfo[]): PackageInfo[] { // ── Pack + hash ──────────────────────────────────────────────────── async function packPackage(pkg: PackageInfo): Promise { - // npm pack outputs the tarball filename to stdout - const tarballName = execSync("npm pack --pack-destination .", { + // npm >= 10 forwards lifecycle-script (prepack) stdout into `npm pack`'s stdout, + // so the filename is discovered from the filesystem instead of stdout. + const packDir = await mkdtemp(join(tmpdir(), "aws-blocks-pack-")); + + execSync(`npm pack --pack-destination ${JSON.stringify(packDir)}`, { cwd: pkg.dirPath, encoding: "utf-8", - }).trim(); + }); + + const tarballs = readdirSync(packDir).filter((name) => name.endsWith(".tgz")); + if (tarballs.length !== 1) { + throw new Error( + `expected exactly one .tgz in ${packDir} for ${pkg.name}, found ${tarballs.length}: ${tarballs.join(", ")}`, + ); + } - const tarballPath = join(pkg.dirPath, tarballName); + const tarballPath = join(packDir, tarballs[0]!); - // Validate tarball path stays within the package directory + // Validate tarball path stays within the pack destination directory const resolvedTarball = resolve(tarballPath); - if (!resolvedTarball.startsWith(resolve(pkg.dirPath))) { - throw new Error(`Tarball path escapes package directory: ${resolvedTarball}`); + if (!resolvedTarball.startsWith(resolve(packDir))) { + throw new Error(`Tarball path escapes pack destination directory: ${resolvedTarball}`); } const tarballBuf = await readFile(tarballPath); diff --git a/scripts/sync-block-docs.mjs b/scripts/sync-block-docs.mjs deleted file mode 100644 index 2f403e23..00000000 --- a/scripts/sync-block-docs.mjs +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env node -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Assembles `packages/blocks/docs/` from all Building Block READMEs. - * Run at build/publish time (not customer-side). Produces: - * packages/blocks/docs/index.md — decision tree + catalog - * packages/blocks/docs/.md — one per block - */ - -import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, rmSync } from 'node:fs'; -import { join, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const packagesDir = join(__dirname, '..', 'packages'); -const outDir = join(packagesDir, 'blocks', 'docs'); - -const EXCLUDED = new Set(['blocks', 'data-common', 'foundations', 'create-blocks-app']); - -const DECISION_TREE = `# AWS Blocks — Building Block Catalog - -Start from what you need: - -- **Store data** - - Simple key → value (caches, flags, user prefs) → \`KVStore\` ([bb-kv-store](./bb-kv-store.md)) - - Structured records with indexes and queries → \`DistributedTable\` ([bb-distributed-table](./bb-distributed-table.md)) — **default for most data** - - Relational / SQL (joins, transactions) → see [Choosing a data block](#choosing-a-data-block) below - - Files, blobs, uploads, static assets → \`FileBucket\` ([bb-file-bucket](./bb-file-bucket.md)) - - A single config value or secret → \`AppSetting\` ([bb-app-setting](./bb-app-setting.md)) -- **Authenticate users** - - Username/password, prototypes/MVPs → \`AuthBasic\` ([bb-auth-basic](./bb-auth-basic.md)) - - Cognito user pools, MFA, groups → \`AuthCognito\` ([bb-auth-cognito](./bb-auth-cognito.md)) - - External identity provider (OIDC) → \`AuthOIDC\` ([bb-auth-oidc](./bb-auth-oidc.md)) -- **Run work outside the request/response** - - Fire-and-forget background jobs → \`AsyncJob\` ([bb-async-job](./bb-async-job.md)) - - Scheduled / recurring tasks → \`CronJob\` ([bb-cron-job](./bb-cron-job.md)) -- **Push live updates to browsers** (chat, presence, dashboards) → \`Realtime\` ([bb-realtime](./bb-realtime.md)) -- **Build AI features** - - Agent with tool use + conversation → \`Agent\` ([bb-agent](./bb-agent.md)) - - Semantic document retrieval (RAG) → \`KnowledgeBase\` ([bb-knowledge-base](./bb-knowledge-base.md)) -- **Send transactional email** → \`EmailClient\` ([bb-email-client](./bb-email-client.md)) -- **Serve a raw HTTP endpoint** (webhook receiver, health check, redirect, non-JSON response) → \`RawRoute\` ([core](./core.md#rawroute)); everything else goes through \`ApiNamespace\` RPC -- **Observe and operate** - - Structured logs → \`Logger\` ([bb-logger](./bb-logger.md)) - - Custom metrics → \`Metrics\` ([bb-metrics](./bb-metrics.md)) - - Distributed traces → \`Tracer\` ([bb-tracer](./bb-tracer.md)) - - Auto CloudWatch dashboard → \`Dashboard\` ([bb-dashboard](./bb-dashboard.md)) - -### Choosing a data block - -Default to \`DistributedTable\` for your data models unless your domain specifically requires SQL engine capabilities. - -Reach for one of the SQL blocks when you need to filter or join results across more than one related record, filter models on many dimensions with no preset hierarchy, store large objects, require transactions, or otherwise need the flexibility or familiarity of SQL that NoSQL does not offer. - -If you need SQL, prefer \`DistributedDatabase\` for basic Postgres-compatible querying. Use \`Database\` specifically when you need a full (more expensive) Postgres implementation where the engine itself provides and enforces foreign keys, row level security, triggers, views, large transactions (more than 3,000 rows), or integration with an existing Postgres database. Note it carries an idle cost at minimum 0.5 ACU, or a cold start when scaling from zero, unlike the other two blocks.`; - -// Clean and recreate -rmSync(outDir, { recursive: true, force: true }); -mkdirSync(outDir, { recursive: true }); - -// Gather all @aws-blocks packages with READMEs -const packages = readdirSync(packagesDir).filter( - (name) => !name.startsWith('.') && !EXCLUDED.has(name) && existsSync(join(packagesDir, name, 'README.md')), -); - -const catalog = []; - -for (const pkg of packages) { - const content = readFileSync(join(packagesDir, pkg, 'README.md'), 'utf-8'); - writeFileSync(join(outDir, `${pkg}.md`), content); - catalog.push({ pkg, blurb: extractBlurb(content), keywords: extractKeywords(content) }); -} - -catalog.sort((a, b) => a.pkg.localeCompare(b.pkg)); -writeFileSync(join(outDir, 'index.md'), renderIndex(catalog)); - -console.log(`Synced ${catalog.length} block docs → packages/blocks/docs/`); - -// ─── Helpers ───────────────────────────────────────────────────────────────── - -function extractBlurb(content) { - const lines = content.split('\n'); - const h1 = lines.findIndex((l) => l.startsWith('# ')); - if (h1 === -1) return ''; - for (let i = h1 + 1; i < lines.length; i++) { - const line = lines[i].trim(); - if (!line) continue; - if (line.startsWith('#') || line.startsWith('` / `` markers — it + * leaves the rest of the README (the static decision-tree prose, etc.) untouched + * and does NOT generate the shipped `docs/` artifact. That artifact (per-block + * docs// folders + docs/README.md copy) is produced by + * scripts/gen-block-docs.mjs, run as the packages/blocks `prebuild` and `prepack` hooks. + * + * Two modes: + * + * --write (default; `npm run sync-docs`, run MANUALLY when adding/removing a block) + * Render the catalog table from the discovered blocks and inject it between + * the markers in packages/blocks/README.md. Nothing else in the README changes. + * + * --check (CI / PR gate; `npm run sync-docs:check`) + * Render the catalog table in memory and compare it to what is committed + * between the markers. Exit 1 with an actionable message if they differ or the + * markers are missing; exit 0 if in sync. Writes nothing. + * + * Inclusion rule: every package under packages/ that has a README.md and is not in + * EXCLUDED. + * + * NOTE: EXCLUDED + getPackages() are intentionally duplicated in this file and in + * gen-block-docs.mjs so each script stays dependency-free and independently + * runnable (no shared module to resolve, no build step). They MUST agree on the + * block set — keep the two in sync when editing. If this pair grows further, + * extract a shared module instead. + */ + +import { readFileSync, writeFileSync, readdirSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const packagesDir = join(__dirname, '..', 'packages'); +const readmePath = join(packagesDir, 'blocks', 'README.md'); + +const EXCLUDED = new Set(['blocks', 'data-common', 'foundations', 'create-blocks-app']); + +const BEGIN_MARKER = ''; +const END_MARKER = ''; + +const SYNC_HINT = 'Run `npm run sync-docs` and commit the result.'; + +const mode = process.argv.includes('--check') ? 'check' : 'write'; + +const packages = getPackages(); +const catalog = buildCatalog(packages); +const table = renderCatalogTable(catalog); + +if (mode === 'check') { + runCheck(); +} else { + runWrite(); +} + +// ─── Modes ─────────────────────────────────────────────────────────────────── + +function runCheck() { + const readme = readFileSync(readmePath, 'utf-8'); + const current = extractBetweenMarkers(readme); + + if (current === null) { + process.stderr.write( + `❌ Building Block catalog markers (${BEGIN_MARKER} / ${END_MARKER}) not found in packages/blocks/README.md. ${SYNC_HINT}\n`, + ); + process.exit(1); + } + + if (current.trim() !== table.trim()) { + process.stderr.write( + `❌ Building Block catalog in packages/blocks/README.md is out of date. ${SYNC_HINT}\n`, + ); + process.exit(1); + } + + console.log('✅ Building Block catalog in packages/blocks/README.md is up to date.'); + process.exit(0); +} + +function runWrite() { + const readme = readFileSync(readmePath, 'utf-8'); + const updated = injectCatalog(readme, table); + if (updated !== readme) writeFileSync(readmePath, updated); + console.log(`Synced ${catalog.length} blocks → packages/blocks/README.md catalog`); +} + +// ─── Catalog ───────────────────────────────────────────────────────────────── + +function getPackages() { + return readdirSync(packagesDir).filter( + (name) => !name.startsWith('.') && !EXCLUDED.has(name) && existsSync(join(packagesDir, name, 'README.md')), + ); +} + +function buildCatalog(pkgs) { + const entries = pkgs.map((pkg) => { + const readme = readFileSync(join(packagesDir, pkg, 'README.md'), 'utf-8'); + return { pkg, blurb: extractBlurb(readme), keywords: extractKeywords(readme) }; + }); + entries.sort((a, b) => (a.pkg < b.pkg ? -1 : a.pkg > b.pkg ? 1 : 0)); + return entries; +} + +function renderCatalogTable(entries) { + const rows = entries.map( + (e) => `| ${e.pkg} | ${escapeCell(e.blurb) || '—'} | ${escapeCell(e.keywords) || '—'} |`, + ); + return ['| Block | What it does | Keywords |', '|-------|--------------|----------|', ...rows].join('\n'); +} + +function escapeCell(value) { + return (value || '').replace(/\|/g, '\\|'); +} + +// ─── Marker helpers ────────────────────────────────────────────────────────── + +function injectCatalog(readme, catalogTable) { + const begin = readme.indexOf(BEGIN_MARKER); + const end = readme.indexOf(END_MARKER); + if (begin === -1 || end === -1 || end < begin) { + throw new Error( + `Building Block catalog markers (${BEGIN_MARKER} / ${END_MARKER}) not found in packages/blocks/README.md. ` + + 'Add them where the catalog table should live, then re-run.', + ); + } + const before = readme.slice(0, begin + BEGIN_MARKER.length); + const after = readme.slice(end); + return `${before}\n${catalogTable}\n${after}`; +} + +function extractBetweenMarkers(readme) { + const begin = readme.indexOf(BEGIN_MARKER); + const end = readme.indexOf(END_MARKER); + if (begin === -1 || end === -1 || end < begin) return null; + return readme.slice(begin + BEGIN_MARKER.length, end); +} + +// ─── README parsing ────────────────────────────────────────────────────────── + +function extractBlurb(content) { + const lines = content.split('\n'); + const h1 = lines.findIndex((l) => l.startsWith('# ')); + if (h1 === -1) return ''; + for (let i = h1 + 1; i < lines.length; i++) { + const line = lines[i].trim(); + if (!line) continue; + if (line.startsWith('#') || line.startsWith('