diff --git a/.changeset/30091-data-key-trial-page.md b/.changeset/30091-data-key-trial-page.md new file mode 100644 index 00000000000..4db447a42f7 --- /dev/null +++ b/.changeset/30091-data-key-trial-page.md @@ -0,0 +1,16 @@ +--- +'@objectstack/cli': patch +'@objectstack/service-settings': patch +--- + +fix(service-settings): when two data-key files hold different keys, the boot judges the newest stored secrets as a page, not only the newest one, and reports a database split between the two keys + +Clause-②: no — an internal boot heuristic and its notice text. The shapes it changes (`StoredSecretReader` now answers the page, `KeyFileChoice` gains the `mixed` reason and a `tally`, `DataKeyOpensNothingError` gains `judged`) were added by #30042, whose note is still pending, so they first ship together in this release; no released export changes. +Metadata-protocol: none + +- **`@objectstack/service-settings`.** When the shared key file and the project's own key file hold different keys, the boot now trial-opens each of the newest stored `sys_secret` rows (up to ten) under both keys and reads with the key that opens more of them, on a tie the one that opens the newest. A row that opens under neither key (a corrupt or foreign row) no longer decides the boot: if an older row opens under one key, that key is used, and only a page where no row opens under either key keeps the production refusal. When each key opens some of the rows, the database holds secrets under both keys: the boot uses the key that opens more of them and prints an error naming both files, how many rows each key opened, and the fix. The boot's notices now say what was measured ("opens the 3 newest stored secrets", "opens 2 of the 3 newest stored secrets") instead of claiming the chosen file opens all of the database's stored secrets. +- **`@objectstack/cli`.** The one-shot commands (`os migrate`, `os secret rewrap`, `os secret orphans`, `os storage orphans`) share the same trial, so they make the same choice. + +**The remedy for a split database.** Keep a copy of both files, then save again each secret that reads as unset: it is sealed under the key the boot uses. Once nothing reads as unset, put that key at the shared file (or set `OS_SECRET_KEY` to it). + +Supersedes, in the same release: the note for #30042 says the boot opens the newest stored secret under each key and quotes the refusal as "neither data key opens this database's stored secrets". After this change the boot judges the newest rows as a page, and the refusal reads "neither data key opens any of the N newest stored secrets". diff --git a/content/docs/deployment/backup-restore.mdx b/content/docs/deployment/backup-restore.mdx index 2914ade0a69..1b266416b10 100644 --- a/content/docs/deployment/backup-restore.mdx +++ b/content/docs/deployment/backup-restore.mdx @@ -29,6 +29,22 @@ Escrow the key in your secret manager the day you generate it, and treat key + database as one backup unit. +**Two key files on one host.** Without `OS_SECRET_KEY` (or `OS_DEV_CRYPTO_KEY`), +the data key comes from a `dev-crypto-key` file: the canonical one under the +ObjectStack home dir (`OS_HOME`, else `~/.objectstack`), which `os dev` and +`os start` share, or a project's own `/.objectstack/dev-crypto-key`, +left by an earlier `os start`, when that is the only file holding a key. When +both files exist and hold **different** keys, the boot trial-opens the newest +stored secrets (up to ten rows) under each key once the database is open, uses +the file whose key opens more of them, and prints which file it chose and how +many rows each key opened. If each key opens some of the rows, the database +holds secrets under both keys: the boot prints an error naming both files, and +the secrets under the unused key read as unset until they are saved again. If +neither key opens any of them, a production boot refuses to start and names +both files. To converge on one key, keep a copy of both files, save again any +secret that reads as unset, then put the key that opens them at the canonical +path (or set `OS_SECRET_KEY` to it) and back that one key up with the database. + ## Backing up the database Use the native tooling of whatever `OS_DATABASE_URL` points at — ObjectStack diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 40ebc08669a..60570e3ec1a 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -5320,7 +5320,7 @@ export default class Serve extends Command { }; // Fail-closed binder over the shared dev crypto provider. If the - // provider can't be created, leave `secrets` undefined — the plugin + // binder can't be built, leave `secrets` undefined — the plugin // then rejects secret-bearing create/update instead of storing // cleartext (by design). let secrets: any = undefined; @@ -5341,12 +5341,14 @@ export default class Serve extends Command { cryptoProvider: sharedCryptoProvider, }); } catch (cryptoErr: any) { - // Best-effort fail-closed: leave `secrets` undefined so the plugin - // rejects secret-bearing create/update rather than storing - // cleartext. A production deployment with no stable key still - // aborts boot loudly at the secret-field wiring below (where - // LocalCryptoProvider's "Refusing to start in production" error is - // rethrown), so we don't duplicate that abort here. + // Reached only when the module or the binder fails to load: the + // deferred provider constructs nothing here, so a key problem + // never lands in this catch. Leave `secrets` undefined so the + // plugin rejects secret-bearing create/update rather than storing + // cleartext. A key problem surfaces where the provider is first + // built, the secret-field wiring below (`resolve()`), which + // rethrows a production "Refusing to start in production" error, + // so the boot still refuses and nothing is duplicated here. console.warn( chalk.yellow( ` ⚠ datasource admin: no CryptoProvider (${cryptoErr?.message ?? cryptoErr}); secret-bearing datasource create/update will fail closed`, diff --git a/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts b/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts index 9f364b9161b..52840fa0e32 100644 --- a/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts +++ b/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts @@ -123,7 +123,7 @@ describe('#29934 — the key file an earlier `os start` kept in the project', () ] as const)('%s: the child reads its secret back with the %s key', async (_shape, sealer) => { const keys = { legacy: writeKey(legacyFile()), canonical: writeKey(canonicalFile()) }; const handle = await new LocalCryptoProvider({ key: keys[sealer] }).encrypt('sk-30042', ctx); - const stored = async () => ({ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }); + const stored = async () => [{ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }]; for (const { child } of [await startChild(), await devChild()]) { const quiet = vi.spyOn(console, 'warn').mockImplementation(() => {}); // What the child's settings service does at start(), the database open. diff --git a/packages/cli/src/utils/one-shot-settings.pin.test.ts b/packages/cli/src/utils/one-shot-settings.pin.test.ts index 0f18d040d88..dc1ecb92101 100644 --- a/packages/cli/src/utils/one-shot-settings.pin.test.ts +++ b/packages/cli/src/utils/one-shot-settings.pin.test.ts @@ -255,7 +255,7 @@ describe('[#30042] one-shot commands take the same key file, by the same resolut // What the boot's settings service does at start(), the database open. await settleKeyFileChannel({ env, - readStoredSecret: async () => ({ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }), + readStoredSecret: async () => [{ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }], }); expect(await key.provider!.decrypt(handle, ctx)).toBe('sk-start'); expect(key.provider!.keySource).toBe('file'); diff --git a/packages/cli/src/utils/one-shot-settings.ts b/packages/cli/src/utils/one-shot-settings.ts index 9790ddb0675..ef75b52a8fe 100644 --- a/packages/cli/src/utils/one-shot-settings.ts +++ b/packages/cli/src/utils/one-shot-settings.ts @@ -59,6 +59,12 @@ export interface ExistingDataKey { /** * Resolve the data key this host already has, never minting one. A key that * is missing, or set but unusable, is an answer here, not a throw. + * + * ⚠ It writes the key-file channels (`KEY_FILE_CHANNEL_ENV`, + * `KEY_FILE_CANDIDATE_CHANNEL_ENV`) on `opts.env`, which defaults to the real + * `process.env`: it sets or deletes both, so this process and every child it + * spawns afterwards read the file this call resolved. Pass an env map of your + * own to keep `process.env` untouched (the tests do). */ export async function resolveExistingDataKey( opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, @@ -77,11 +83,11 @@ export async function resolveExistingDataKey( // [#30042] The SAME file a `serve` child of `os dev` / `os start` reads for // this project, chosen by the same resolution: the canonical file, the // project's legacy file when it alone holds a key, and — when the two hold - // different keys — whichever opens a stored secret, ruled by the settings - // service in this run's own boot (`settleKeyFileChannel`). This process - // opens the database itself, so it is its own parent: it sets both channels - // on its own env, exactly as `applyCryptoKeyFileChannel` sets them on a - // child's. + // different keys — whichever opens more of the newest stored secrets, ruled + // by the settings service in this run's own boot (`settleKeyFileChannel`). + // This process opens the database itself, so it is its own parent: it sets + // both channels on its own env, exactly as `applyCryptoKeyFileChannel` sets + // them on a child's. const startHome = resolveStartHome(undefined, { hasProjectConfig: existsSync(path.resolve(cwd, 'objectstack.config.ts')), cwd, diff --git a/packages/services/service-settings/README.md b/packages/services/service-settings/README.md index 910b982ce0d..a03486882cb 100644 --- a/packages/services/service-settings/README.md +++ b/packages/services/service-settings/README.md @@ -52,9 +52,18 @@ The default is `LocalCryptoProvider` — AES-256-GCM keyed off a single across every restart and every node, or previously-encrypted secrets become undecryptable. 2. `OS_DEV_CRYPTO_KEY` — dev convenience key (same format). -3. A persisted file at `~/.objectstack/dev-crypto-key` (mode 0600). In +3. A persisted file, `dev-crypto-key` (mode 0600) under `OS_HOME` (legacy + `OBJECTSTACK_HOME`, a leading `~` expanded), else `~/.objectstack`. In development this is auto-created so single-host dev loops survive - restarts; in production it is only *read*, never minted. + restarts; in production it is only *read*, never minted. Under the CLI + (`os dev`, `os start`, the one-shot commands) a project whose own + `.objectstack/dev-crypto-key` (left by an earlier `os start`) is the only + file holding a key reads that one instead. When both files hold + *different* keys, the boot trial-opens the newest stored `sys_secret` + rows (up to ten) under each key once the database is open, and reads the + file whose key opens more of them; a split between the two keys is a boot + error naming both files, and a production boot where neither key opens any + of them refuses to start (`DataKeyOpensNothingError`). **Fail-loud in production.** When `NODE_ENV=production` and no stable key source (env var or pre-existing file) is available, the provider refuses diff --git a/packages/services/service-settings/src/index.ts b/packages/services/service-settings/src/index.ts index e68e856bd93..ba9b6356c56 100644 --- a/packages/services/service-settings/src/index.ts +++ b/packages/services/service-settings/src/index.ts @@ -38,7 +38,7 @@ export { type PersistedKeyFile, KEY_FILE_CHANNEL_ENV, // When those two files hold different keys, the process that opens the - // database rules between them by trial-opening a stored secret, or refuses. + // database rules between them by trial-opening its newest stored secrets, or refuses. KEY_FILE_CANDIDATE_CHANNEL_ENV, chooseKeyFileByTrialOpen, settleKeyFileChannel, diff --git a/packages/services/service-settings/src/local-crypto-provider.test.ts b/packages/services/service-settings/src/local-crypto-provider.test.ts index f843e4aabec..5f5958f07bb 100644 --- a/packages/services/service-settings/src/local-crypto-provider.test.ts +++ b/packages/services/service-settings/src/local-crypto-provider.test.ts @@ -271,7 +271,8 @@ describe('#30042 — two key files: the boot takes the one that opens a stored s const h = await new LocalCryptoProvider({ key }).encrypt('sk-30042', { scope, namespace: 'ai', key: 'openai_api_key' }); return { namespace: 'ai', key: 'openai_api_key', ciphertext: h.ciphertext }; }; - const reader = (sample: StoredSecretSample | undefined) => async () => sample; + /** A reader returning this page of stored rows, newest first. */ + const reader = (...page: StoredSecretSample[]) => async () => page; it('the case-1 shape: a start-sealed database with a foreign home key reads with the legacy key', async () => { writeKey(canonical()); // `os dev` in another project minted this one @@ -343,7 +344,7 @@ describe('#30042 — two key files: the boot takes the one that opens a stored s it('no stored secret: nothing to judge, the canonical file stands', async () => { writeKey(canonical()); writeKey(legacy()); - const choice = await chooseKeyFileByTrialOpen({ answer: bothDiffer(), readStoredSecret: reader(undefined), mode: 'production' }); + const choice = await chooseKeyFileByTrialOpen({ answer: bothDiffer(), readStoredSecret: reader(), mode: 'production' }); expect(choice).toMatchObject({ path: canonical(), reason: 'no-stored-secret' }); }); @@ -362,8 +363,101 @@ describe('#30042 — two key files: the boot takes the one that opens a stored s expect(choice.notice?.message).toContain('connection reset'); }); + describe('#30091 — the trial scores both keys over the whole page, not its newest row', () => { + it.each(['production', 'development'] as const)( + '%s: rows split between the keys is a mixed-key boot ERROR naming both paths, on the key that opens more', + async (mode) => { + const canonicalKey = writeKey(canonical()); + const legacyKey = writeKey(legacy()); + // Newest first: the newest row is the canonical key's, the three older ones the legacy key's. + const page = [await sealed(canonicalKey), await sealed(legacyKey), await sealed(legacyKey), await sealed(legacyKey)]; + const choice = await chooseKeyFileByTrialOpen({ answer: bothDiffer(), readStoredSecret: reader(...page), mode }); + expect(choice).toMatchObject({ + path: legacy(), + reason: 'mixed', + other: canonical(), + tally: { judged: 4, chosen: 3, other: 1 }, + }); + expect(choice.notice?.level).toBe('error'); + expect(choice.notice?.message).toContain(canonical()); + expect(choice.notice?.message).toContain(legacy()); + }, + ); + + it.each([ + ['canonical', 'legacy'], + ['legacy', 'canonical'], + ] as const)('a tie goes to the key that opens the newest row (%s newest)', async (newest, older) => { + const keys = { canonical: writeKey(canonical()), legacy: writeKey(legacy()) }; + const files = { canonical: canonical(), legacy: legacy() }; + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(keys[newest]), await sealed(keys[older])), + mode: 'production', + }); + expect(choice).toMatchObject({ path: files[newest], reason: 'mixed', tally: { judged: 2, chosen: 1, other: 1 } }); + }); + + it('a corrupt newest row no longer refuses a production boot: an older row settles the choice', async () => { + writeKey(canonical()); + const startKey = writeKey(legacy()); + const corrupt = await sealed(randomBytes(32)); // opens under neither key + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(corrupt, await sealed(startKey), await sealed(startKey, 'object_secret_field')), + mode: 'production', + }); + expect(choice).toMatchObject({ path: legacy(), reason: 'opens', tally: { judged: 3, chosen: 2, other: 0 } }); + expect(choice.notice?.level).toBe('warn'); + }); + + it('one key opening every judged row is `opens`, with the count it was measured on', async () => { + const devKey = writeKey(canonical()); + writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(devKey), await sealed(devKey, 'datasource_credential')), + mode: 'production', + }); + expect(choice).toMatchObject({ path: canonical(), reason: 'opens', tally: { judged: 2, chosen: 2, other: 0 } }); + }); + + it('a page where no row opens under either key keeps the production refusal, counting the page', async () => { + writeKey(canonical()); + writeKey(legacy()); + const refusal = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(randomBytes(32)), await sealed(randomBytes(32))), + mode: 'production', + }).catch((e: unknown) => e); + expect(refusal).toBeInstanceOf(DataKeyOpensNothingError); + expect((refusal as DataKeyOpensNothingError).judged).toBe(2); + expect((refusal as Error).message).toContain('Refusing to start in production'); + }); + + it('settleKeyFileChannel: a mixed page settles on the majority key and says so as console.error, once', async () => { + const canonicalKey = writeKey(canonical()); + const legacyKey = writeKey(legacy()); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const env: Record = { + NODE_ENV: 'production', + [KEY_FILE_CHANNEL_ENV]: canonical(), + [KEY_FILE_CANDIDATE_CHANNEL_ENV]: legacy(), + }; + const page = [await sealed(legacyKey), await sealed(canonicalKey), await sealed(canonicalKey)]; + const choice = await settleKeyFileChannel({ env, readStoredSecret: reader(...page) }); + expect(choice).toMatchObject({ path: canonical(), reason: 'mixed' }); + expect(env[KEY_FILE_CHANNEL_ENV]).toBe(canonical()); + expect(env[KEY_FILE_CANDIDATE_CHANNEL_ENV]).toBeUndefined(); + expect(error).toHaveBeenCalledTimes(1); + expect(String(error.mock.calls[0]?.[0])).toContain(legacy()); + expect(warn).not.toHaveBeenCalled(); + }); + }); + it('every one-file answer passes through untried', async () => { - const read = vi.fn(async () => undefined); + const read = vi.fn(async () => [] as StoredSecretSample[]); for (const answer of [ { path: canonical(), kind: 'canonical', canonical: canonical() }, { path: legacy(), kind: 'legacy-only', canonical: canonical(), legacy: legacy() }, @@ -431,7 +525,7 @@ describe('#30042 — two key files: the boot takes the one that opens a stored s }); it('nothing to settle without a candidate', async () => { - const read = vi.fn(reader(undefined)); + const read = vi.fn(reader()); expect(await settleKeyFileChannel({ env: { [KEY_FILE_CHANNEL_ENV]: canonical() }, readStoredSecret: read })).toBeUndefined(); expect(read).not.toHaveBeenCalled(); }); @@ -460,12 +554,17 @@ describe('#30042 — two key files: the boot takes the one that opens a stored s describe('#30042 — the reader over the engine', () => { const v2 = 'v2:' + randomBytes(40).toString('base64'); - it('reads the newest rows as the system and skips a marker the provider does not know', async () => { + it('reads the newest page as the system, keeps its order, and skips a marker the provider does not know', async () => { + const older = 'v2:' + randomBytes(40).toString('base64'); const find = vi.fn(async () => [ { namespace: 'x', key: 'y', ciphertext: 'v9:abc' }, { namespace: 'ai', key: 'openai_api_key', ciphertext: v2 }, + { namespace: 'mail', key: 'smtp_password', ciphertext: older }, + ]); + expect(await storedSecretReaderOverEngine({ find })()).toEqual([ + { namespace: 'ai', key: 'openai_api_key', ciphertext: v2 }, + { namespace: 'mail', key: 'smtp_password', ciphertext: older }, ]); - expect(await storedSecretReaderOverEngine({ find })()).toEqual({ namespace: 'ai', key: 'openai_api_key', ciphertext: v2 }); expect(find).toHaveBeenCalledWith('sys_secret', expect.objectContaining({ orderBy: [{ field: 'created_at', order: 'desc' }], context: { isSystem: true }, @@ -474,7 +573,7 @@ describe('#30042 — the reader over the engine', () => { it('a table that does not exist holds no row; any other failure rejects', async () => { const missing = storedSecretReaderOverEngine({ find: async () => { throw new Error('no such table: sys_secret'); } }); - expect(await missing()).toBeUndefined(); + expect(await missing()).toEqual([]); const broken = storedSecretReaderOverEngine({ find: async () => { throw new Error('connection reset'); } }); await expect(broken()).rejects.toThrow('connection reset'); }); diff --git a/packages/services/service-settings/src/local-crypto-provider.ts b/packages/services/service-settings/src/local-crypto-provider.ts index ff1d4147256..e1dfe835d0b 100644 --- a/packages/services/service-settings/src/local-crypto-provider.ts +++ b/packages/services/service-settings/src/local-crypto-provider.ts @@ -635,19 +635,21 @@ export interface StoredSecretSample { } /** - * Reads ONE stored ciphertext to judge a key by, or `undefined` when the + * Reads the stored ciphertexts to judge a key by, NEWEST FIRST: the page + * {@link chooseKeyFileByTrialOpen} scores both keys over. Empty when the * database holds none. A failed read rejects: it is not an empty table. */ -export type StoredSecretReader = () => Promise; +export type StoredSecretReader = () => Promise; -/** The newest rows a reader looks through for one whose marker this provider knows. */ +/** How many of the newest rows the trial reads; it judges those of them whose marker this provider knows. */ const TRIAL_SAMPLE_PAGE = 10; /** - * The {@link StoredSecretReader} over a data engine: the NEWEST `sys_secret` - * row (by `created_at`) whose ciphertext records a derivation this provider - * knows, read as the system, outside any tenant. The newest row is the one the - * deployment sealed last, so the key that opens it is the key in use. + * The {@link StoredSecretReader} over a data engine: the newest + * {@link TRIAL_SAMPLE_PAGE} `sys_secret` rows (by `created_at`), read as the + * system, outside any tenant, less every row whose ciphertext records a + * derivation this provider does not know (no key could open it, so it says + * nothing about the keys). * * A table that does not exist yet (a database that has never booted) holds no * row. ⛔ Any other failed read rejects: it says nothing about the keys. @@ -665,19 +667,20 @@ export function storedSecretReaderOverEngine(engine: { context: { isSystem: true }, }); } catch (error) { - if (isMissingTableError(error, 'sys_secret')) return undefined; + if (isMissingTableError(error, 'sys_secret')) return []; throw error; } const rows = (Array.isArray(answer) ? answer : (answer as { data?: unknown[] } | null)?.data) ?? []; + const page: StoredSecretSample[] = []; for (const row of rows as Array>) { if (ciphertextDerivationStatus(row?.ciphertext) === 'unknown') continue; - return { + page.push({ namespace: String(row.namespace ?? ''), key: String(row.key ?? ''), ciphertext: row.ciphertext as string, - }; + }); } - return undefined; + return page; }; } @@ -712,6 +715,25 @@ async function keyOpensSample(key: Buffer, sample: StoredSecretSample): Promise< return false; } +/** + * Which rows of the page `key` opens, newest first: their count, and the + * position of the newest one it opens (`-1` for none, or no key). + */ +async function scoreKeyOverPage( + key: Buffer | undefined, + page: readonly StoredSecretSample[], +): Promise<{ opens: number; newest: number }> { + let opens = 0; + let newest = -1; + if (!key) return { opens, newest }; + for (const [index, sample] of page.entries()) { + if (!(await keyOpensSample(key, sample))) continue; + opens += 1; + if (newest < 0) newest = index; + } + return { opens, newest }; +} + /** * Which key file a boot reads, and why — {@link chooseKeyFileByTrialOpen}'s * answer. @@ -720,23 +742,40 @@ async function keyOpensSample(key: Buffer, sample: StoredSecretSample): Promise< * second key to rule out. * - `no-stored-secret` — two keys, but the database holds no ciphertext to * judge them by: the canonical file, as before. - * - `opens` — the chosen file's key opens the newest stored ciphertext and the - * other's does not. - * - `opens-nothing` — neither key opens it (development: the canonical file, - * with a boot error; production refuses instead, see + * - `opens` — the chosen file's key opens at least one row of the judged page + * and the other's opens none. + * - `mixed` — each key opens at least one row of the page: the database holds + * secrets under both. The chosen file opens more of them (on a tie, the + * newest of them); the rows under the other read as unset, and the boot + * says so as an error naming both paths. + * - `opens-nothing` — neither key opens any row of the page (development: the + * canonical file, with a boot error; production refuses instead, see * {@link DataKeyOpensNothingError}). * - `not-judged` — the stored ciphertext could not be read: the canonical * file, with a boot error naming the read. */ export interface KeyFileChoice { path: string; - reason: 'single' | 'no-stored-secret' | 'opens' | 'opens-nothing' | 'not-judged'; + reason: 'single' | 'no-stored-secret' | 'opens' | 'mixed' | 'opens-nothing' | 'not-judged'; /** The file not chosen, when there were two. */ other?: string; + /** + * What the trial measured, when it opened rows: the page it judged, and how + * many rows of it each file's key opens. Absent when nothing was judged. + */ + tally?: { judged: number; chosen: number; other: number }; /** What the boot prints, and at which level; absent for `single`. */ notice?: { level: 'warn' | 'error'; message: string }; } +/** `the newest stored secret` / `the N newest stored secrets`: the page the trial judged, as the notices name it. */ +const newestStored = (judged: number): string => + judged === 1 ? 'the newest stored secret' : `the ${judged} newest stored secrets`; + +/** `opens the newest stored secret` / `opens 3 of the 10 newest stored secrets`: what a key was measured to open. */ +const opensOf = (opens: number, judged: number): string => + opens === judged ? `opens ${newestStored(judged)}` : `opens ${opens} of ${newestStored(judged)}`; + const OPENS_NOTHING_CONSEQUENCE = `Every stored secret (encrypted settings, secret fields, datasource credentials) reads back as unset under either key.`; @@ -745,20 +784,22 @@ const opensNothingRemedy = (canonical: string): string => `files first), or set ${SECRET_KEY_ENV} to it.`; /** - * A production boot whose two candidate key files both fail to open the - * database's newest stored secret. The fail-loud contract this provider keeps - * (see "Fail-loud guarantee" above): production refuses to run under a key it - * cannot vouch for, because every secret would read as unset while the server - * looks healthy. + * A production boot whose two candidate key files both fail to open every + * stored secret the trial judged (the newest rows). The fail-loud contract + * this provider keeps (see "Fail-loud guarantee" above): production refuses to + * run under a key it cannot vouch for, because every secret would read as + * unset while the server looks healthy. */ export class DataKeyOpensNothingError extends Error { constructor( readonly canonical: string, readonly legacy: string, + /** How many stored secrets the trial judged (the newest rows). */ + readonly judged = 1, ) { super( - `[LocalCryptoProvider] Refusing to start in production: neither data key opens this database's ` + - `stored secrets.\n` + + `[LocalCryptoProvider] Refusing to start in production: neither data key opens any of ` + + `${newestStored(judged)}.\n` + ` Tried:\n ${canonical}\n ${legacy}\n` + ` ${OPENS_NOTHING_CONSEQUENCE}\n` + ` ${opensNothingRemedy(canonical)}`, @@ -769,19 +810,29 @@ export class DataKeyOpensNothingError extends Error { /** * Rule between the two key files of a `both-differ` answer by the database - * itself: trial-open one stored `sys_secret` ciphertext (the newest one - * `readStoredSecret` returns) under each file's key, and take the file whose - * key opens it. The canonical file is tried first, so a ciphertext both keys - * open (impossible for two different AES keys, short of a forgery) keeps it. + * itself: trial-open the newest stored `sys_secret` ciphertexts (the page + * `readStoredSecret` returns) under each file's key, count the rows each key + * opens, and take the file whose key opens more of them. + * + * - One key opens rows, the other none: that key's file (`opens`). A row that + * opens under neither (a corrupt or foreign row) does not change that, so a + * corrupt newest row no longer decides the boot when an older row settles it. + * - Both keys open rows (`mixed`): the file whose key opens more, on a tie the + * one that opens the newest of them (the key sealed with last). The rows + * under the other key read as unset under either choice, so the notice is an + * `error` naming both paths, both counts and the fix, in every posture: the + * boot is measured, not guessed, and refusing it would leave the operator no + * running server to re-save the minority's secrets through. + * - Neither key opens any row: production throws + * {@link DataKeyOpensNothingError}; development and test keep the canonical + * file and return an `error` notice, the posture split of the provider's own + * fail-loud contract. * * Every other answer of {@link resolvePersistedKeyFile} names one file and is * returned as it is (`single`). With no stored ciphertext there is nothing to - * judge, and the canonical file stands (`no-stored-secret`). - * - * Neither key opens it: production throws {@link DataKeyOpensNothingError}; - * development and test keep the canonical file and return an `error` notice, - * the posture split of the provider's own fail-loud contract. ⛔ Never a - * silent choice: every answer between two files carries its notice. + * judge, and the canonical file stands (`no-stored-secret`). ⛔ Never a silent + * choice: every answer between two files carries its notice, and each notice + * says what was measured (how many of the newest rows), never more. */ export async function chooseKeyFileByTrialOpen(opts: { answer: PersistedKeyFile; @@ -796,9 +847,9 @@ export async function chooseKeyFileByTrialOpen(opts: { const legacy = answer.legacy; const mode = opts.mode ?? detectMode(opts.env ?? processEnv()); - let sample: StoredSecretSample | undefined; + let page: readonly StoredSecretSample[]; try { - sample = await opts.readStoredSecret(); + page = await opts.readStoredSecret(); } catch (error) { const why = error instanceof Error ? error.message : String(error); return { @@ -816,7 +867,7 @@ export async function chooseKeyFileByTrialOpen(opts: { }; } - if (!sample) { + if (page.length === 0) { return { path: canonical, reason: 'no-stored-secret', @@ -830,47 +881,70 @@ export async function chooseKeyFileByTrialOpen(opts: { }; } - const canonicalKey = loadExistingKey(canonical); - const legacyKey = loadExistingKey(legacy); - if (canonicalKey && (await keyOpensSample(canonicalKey, sample))) { + const judged = page.length; + const c = await scoreKeyOverPage(loadExistingKey(canonical), page); + const l = await scoreKeyOverPage(loadExistingKey(legacy), page); + const neither = judged - c.opens - l.opens; + const neitherNote = neither > 0 ? ` ${neither} of them open under neither key.` : ''; + + if (c.opens === 0 && l.opens === 0) { + if (mode === 'production') throw new DataKeyOpensNothingError(canonical, legacy, judged); return { path: canonical, - reason: 'opens', + reason: 'opens-nothing', other: legacy, + tally: { judged, chosen: 0, other: 0 }, notice: { - level: 'warn', + level: 'error', message: - `[LocalCryptoProvider] Data key: using ${canonical}, which opens this database's stored secrets. ` + - `${legacy} holds a different key that does not, and is not used.`, + `[LocalCryptoProvider] Neither data key opens any of ${newestStored(judged)}. Tried ${canonical} ` + + `(used) and ${legacy}. ${OPENS_NOTHING_CONSEQUENCE} ${opensNothingRemedy(canonical)}`, }, }; } - if (legacyKey && (await keyOpensSample(legacyKey, sample))) { + + // The canonical file keeps a tie only when it also opens the newest of the + // rows either key opens; `newest` is a position, lower is newer. + const legacyWins = + l.opens > c.opens || (l.opens === c.opens && l.newest >= 0 && (c.newest < 0 || l.newest < c.newest)); + const [path, other, chosen, rejected] = legacyWins + ? [legacy, canonical, l, c] + : [canonical, legacy, c, l]; + const tally = { judged, chosen: chosen.opens, other: rejected.opens }; + const converge = + path === legacy + ? ` \`os dev\` and \`os start\` share ${canonical}: keep a copy of it, then move ${legacy} over it to converge.` + : ''; + + if (rejected.opens === 0) { return { - path: legacy, + path, reason: 'opens', - other: canonical, + other, + tally, notice: { level: 'warn', message: - `[LocalCryptoProvider] Data key: using ${legacy}, which opens this database's stored secrets ` + - `(an earlier \`os start\` kept it). ${canonical} holds a different key that does not, and is not ` + - `used. \`os dev\` and \`os start\` share ${canonical}: keep a copy of it, then move ${legacy} over ` + - `it to converge.`, + `[LocalCryptoProvider] Data key: using ${path}, which ${opensOf(chosen.opens, judged)}` + + `${path === legacy ? ' (an earlier `os start` kept it)' : ''}. ${other} holds a different key that ` + + `opens none of them, and is not used.${neitherNote}${converge}`, }, }; } - if (mode === 'production') throw new DataKeyOpensNothingError(canonical, legacy); return { - path: canonical, - reason: 'opens-nothing', - other: legacy, + path, + reason: 'mixed', + other, + tally, notice: { level: 'error', message: - `[LocalCryptoProvider] Neither data key opens this database's stored secrets. Tried ${canonical} ` + - `(used) and ${legacy}. ${OPENS_NOTHING_CONSEQUENCE} ${opensNothingRemedy(canonical)}`, + `[LocalCryptoProvider] Data key: this database holds secrets sealed under two different keys. Of ` + + `${newestStored(judged)}, ${chosen.opens} open under ${path} (used) and ${rejected.opens} under ` + + `${other} (not used).${neitherNote} Every secret sealed under ${other} reads back as unset while the ` + + `server looks healthy. Fix: keep a copy of both files, then save each secret that reads as unset again, ` + + `which seals it under ${path}.${converge}`, }, }; } diff --git a/packages/services/service-settings/src/settings-service-plugin.ts b/packages/services/service-settings/src/settings-service-plugin.ts index c963f2171fb..629b996deee 100644 --- a/packages/services/service-settings/src/settings-service-plugin.ts +++ b/packages/services/service-settings/src/settings-service-plugin.ts @@ -268,9 +268,9 @@ export class SettingsServicePlugin implements Plugin { // consumer has decrypted yet (they all read at `kernel:ready` or later, so // every provider they build reads the chosen file). A no-op unless the CLI // parent named a second candidate (`settleKeyFileChannel`). A production - // boot where neither key opens the stored secrets throws here, which stops - // the boot: the provider's fail-loud contract, at the one place that can - // judge it. + // boot where neither key opens any of the newest stored secrets throws + // here, which stops the boot: the provider's fail-loud contract, at the one + // place that can judge it. let settleEngine: IDataEngine | null = null; try { settleEngine = ctx.getService('objectql'); @@ -280,7 +280,7 @@ export class SettingsServicePlugin implements Plugin { await settleKeyFileChannel({ readStoredSecret: settleEngine ? storedSecretReaderOverEngine(settleEngine as unknown as Parameters[0]) - : async () => undefined, + : async () => [], }); ctx.hook('kernel:ready', async () => {