From 501bddf0eb142138d5c25d3821e609aaf040577f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:17:57 +0000 Subject: [PATCH 1/4] fix(service-settings,cli): a boot with two data-key files picks the one that opens its stored secrets, or refuses When the canonical key file and the project's legacy file hold different keys, the process that opens the database trial-opens the newest stored sys_secret ciphertext under each and reads the file whose key opens it. Production refuses to boot when neither opens it; development prints a boot error. One-shot commands resolve the same file the same way, and a leading ~ in OS_HOME is expanded as the CLI home resolution does. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01WGufzhkoyLG9aL3MMfCgCW --- packages/cli/src/commands/serve.ts | 17 +- .../utils/internal-crypto-key-file-channel.ts | 27 +- packages/cli/src/utils/one-shot-settings.ts | 50 ++- .../services/service-settings/src/index.ts | 12 + .../src/local-crypto-provider.ts | 395 +++++++++++++++++- .../src/settings-service-plugin.ts | 26 +- 6 files changed, 505 insertions(+), 22 deletions(-) diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 887c5f6683b..40ebc08669a 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -5325,13 +5325,17 @@ export default class Serve extends Command { // cleartext (by design). let secrets: any = undefined; try { - const { LocalCryptoProvider } = await import( + const { deferredLocalCryptoProvider } = await import( /* webpackIgnore: true */ '@objectstack/service-settings' ); // First block to touch `sharedCryptoProvider` (still undefined - // here), so create it directly; the secret-field wiring below - // reuses this instance so every sys_secret shares one key. - sharedCryptoProvider = new LocalCryptoProvider(); + // here); the secret-field wiring below reuses this instance so + // every sys_secret shares one key. Built at its FIRST USE, not + // here: the database is not open yet, and when two key files hold + // different keys the boot chooses between them only once it is + // (the settings service's `start()`). The binder first decrypts at + // `kernel:ready`, after that choice. + sharedCryptoProvider = deferredLocalCryptoProvider(); secrets = createDatasourceSecretBinder({ engine: lazySecretEngine, cryptoProvider: sharedCryptoProvider, @@ -5631,6 +5635,11 @@ export default class Serve extends Command { // every sys_secret value after a restart. Let that error be loud: // secret writes must not proceed under an unstable key. sharedCryptoProvider = new LocalCryptoProvider(); + } else if (typeof sharedCryptoProvider.resolve === 'function') { + // The deferred instance the datasource binder shares: build it now + // (the key file was chosen during `runtime.start()`), so the same + // production refusal stays at boot rather than at the first write. + sharedCryptoProvider.resolve(); } dataEngine.setCryptoProvider(sharedCryptoProvider); if (isDev) { diff --git a/packages/cli/src/utils/internal-crypto-key-file-channel.ts b/packages/cli/src/utils/internal-crypto-key-file-channel.ts index 0fd31f9e63c..4004bd5ad29 100644 --- a/packages/cli/src/utils/internal-crypto-key-file-channel.ts +++ b/packages/cli/src/utils/internal-crypto-key-file-channel.ts @@ -22,8 +22,12 @@ import { resolveStartHome } from './start-home.js'; * `resolvePersistedKeyFile` over the env the child sees, with `OS_HOME` as dev * reads it (`--home` counts as `OS_HOME`, being start's spelling of it). The * file `os start` used before — `/dev-crypto-key` — is the legacy - * file: read when it alone holds a key, named in a warning when it and the - * canonical file both hold different keys, never chosen silently. + * file: read when it alone holds a key, and never chosen silently. When it and + * the canonical file hold different keys, the parent cannot rule between them: + * it names the canonical file on the channel and the legacy file on the + * second channel `OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE`, and the child, which + * opens the database, takes the file whose key opens a stored secret (or + * refuses in production when neither does; `settleKeyFileChannel`). * * An env key (`OS_SECRET_KEY`, a usable `OS_DEV_CRYPTO_KEY`) still answers * first, unchanged: no file is named and the channel is removed. The parent @@ -46,7 +50,9 @@ export async function applyCryptoKeyFileChannel( homeFlag?: string; }, ): Promise { - const { resolvePersistedKeyFile, KEY_FILE_CHANNEL_ENV } = await import('@objectstack/service-settings'); + const { resolvePersistedKeyFile, KEY_FILE_CHANNEL_ENV, KEY_FILE_CANDIDATE_CHANNEL_ENV } = await import( + '@objectstack/service-settings' + ); const startHome = resolveStartHome(opts.homeFlag, { hasProjectConfig: opts.hasProjectConfig, cwd: opts.cwd, @@ -57,6 +63,9 @@ export async function applyCryptoKeyFileChannel( env: keyEnv, legacyKeyFile: path.join(startHome, 'dev-crypto-key'), }); + // Both channels are the parent's: an inherited copy never speaks for a file + // this run did not choose. + delete childEnv[KEY_FILE_CANDIDATE_CHANNEL_ENV]; if (!answer) { delete childEnv[KEY_FILE_CHANNEL_ENV]; return undefined; @@ -68,11 +77,15 @@ export async function applyCryptoKeyFileChannel( `\`os dev\` and \`os start\` now share ${answer.canonical}; move the file there to converge.` ); } - if (answer.kind === 'both-differ') { + if (answer.kind === 'both-differ' && answer.legacy) { + // Only the child opens the database, so only the child can tell which key + // opens its stored secrets: it rules between the two files at boot + // (`settleKeyFileChannel` in the settings service), names the one it + // took, and refuses in production when neither opens them. + childEnv[KEY_FILE_CANDIDATE_CHANNEL_ENV] = answer.legacy; return ( - `Data key: two different key files exist, ${answer.canonical} (used) and ${answer.legacy} (NOT used). ` + - `Secrets saved under the unused key will not decrypt. If this project's secrets were saved under ` + - `\`os start\`, keep a copy of both files and move ${answer.legacy} over ${answer.canonical}.` + `Data key: two different key files exist, ${answer.canonical} and ${answer.legacy}. ` + + `The server uses the one that opens this project's stored secrets, and says which.` ); } return undefined; diff --git a/packages/cli/src/utils/one-shot-settings.ts b/packages/cli/src/utils/one-shot-settings.ts index 92522a98145..9790ddb0675 100644 --- a/packages/cli/src/utils/one-shot-settings.ts +++ b/packages/cli/src/utils/one-shot-settings.ts @@ -1,6 +1,9 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +import { existsSync } from 'node:fs'; +import path from 'node:path'; import type { ICryptoProvider } from '@objectstack/spec/contracts'; +import { resolveStartHome } from './start-home.js'; /** * [#21471] The settings service a ONE-SHOT command composes, and the one place @@ -57,16 +60,49 @@ export interface ExistingDataKey { * Resolve the data key this host already has, never minting one. A key that * is missing, or set but unusable, is an answer here, not a throw. */ -export async function resolveExistingDataKey(): Promise { +export async function resolveExistingDataKey( + opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +): Promise { // Loaded at the point of use, never at module load: oclif imports every // command module on every invocation while building its table. - const { LocalCryptoProvider } = await import('@objectstack/service-settings'); + const { + LocalCryptoProvider, + deferredLocalCryptoProvider, + resolvePersistedKeyFile, + KEY_FILE_CHANNEL_ENV, + KEY_FILE_CANDIDATE_CHANNEL_ENV, + } = await import('@objectstack/service-settings'); + const env = opts.env ?? process.env; + const cwd = opts.cwd ?? process.cwd(); + // [#30042] The SAME file a `serve` child of `os dev` / `os start` reads for + // this project, chosen by the same resolution: the canonical file, the + // project's legacy file when it alone holds a key, and — when the two hold + // different keys — whichever opens a stored secret, ruled by the settings + // service in this run's own boot (`settleKeyFileChannel`). This process + // opens the database itself, so it is its own parent: it sets both channels + // on its own env, exactly as `applyCryptoKeyFileChannel` sets them on a + // child's. + const startHome = resolveStartHome(undefined, { + hasProjectConfig: existsSync(path.resolve(cwd, 'objectstack.config.ts')), + cwd, + env, + }); + const answer = resolvePersistedKeyFile({ env, legacyKeyFile: path.join(startHome, 'dev-crypto-key') }); + // The canonical file is the one the provider reads with no channel at all. + delete env[KEY_FILE_CANDIDATE_CHANNEL_ENV]; + if (answer && answer.kind !== 'canonical') env[KEY_FILE_CHANNEL_ENV] = answer.path; + else delete env[KEY_FILE_CHANNEL_ENV]; + const strict = () => ({ mode: 'production' as const, env: { ...env, OS_CRYPTO_AUTOKEY: undefined } }); try { - const provider = new LocalCryptoProvider({ - mode: 'production', - env: { ...process.env, OS_CRYPTO_AUTOKEY: undefined }, - }); - return { provider, unavailable: null }; + // Built now, over the file this answer names: it proves a key exists + // before the boot, and is the provider itself whenever one file is named. + const provider = new LocalCryptoProvider(strict()); + if (answer?.kind !== 'both-differ' || !answer.legacy) return { provider, unavailable: null }; + // Two candidate keys: the provider this run uses is built at its first + // use, after the boot's settings service ruled between them (every reader + // of a stored secret in a one-shot runs at `kernel:ready` or after it). + env[KEY_FILE_CANDIDATE_CHANNEL_ENV] = answer.legacy; + return { provider: deferredLocalCryptoProvider(strict), unavailable: null }; } catch (error) { return { provider: null, unavailable: error instanceof Error ? error.message : String(error) }; } diff --git a/packages/services/service-settings/src/index.ts b/packages/services/service-settings/src/index.ts index 0b7dba07f1a..e68e856bd93 100644 --- a/packages/services/service-settings/src/index.ts +++ b/packages/services/service-settings/src/index.ts @@ -37,6 +37,18 @@ export { resolvePersistedKeyFile, type PersistedKeyFile, KEY_FILE_CHANNEL_ENV, + // When those two files hold different keys, the process that opens the + // database rules between them by trial-opening a stored secret, or refuses. + KEY_FILE_CANDIDATE_CHANNEL_ENV, + chooseKeyFileByTrialOpen, + settleKeyFileChannel, + storedSecretReaderOverEngine, + deferredLocalCryptoProvider, + DataKeyOpensNothingError, + type KeyFileChoice, + type StoredSecretReader, + type StoredSecretSample, + type DeferredLocalCryptoProvider, } from './local-crypto-provider.js'; export { type SettingsActionHandler, diff --git a/packages/services/service-settings/src/local-crypto-provider.ts b/packages/services/service-settings/src/local-crypto-provider.ts index 6c34fe076cc..0735550bbcb 100644 --- a/packages/services/service-settings/src/local-crypto-provider.ts +++ b/packages/services/service-settings/src/local-crypto-provider.ts @@ -11,6 +11,7 @@ import { createHash, createHmac, randomBytes, createCipheriv, createDecipheriv } import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join, resolve as resolvePath } from 'node:path'; +import { isMissingTableError } from '@objectstack/types'; /** * LocalCryptoProvider — the default, KMS-free `ICryptoProvider`. It is an @@ -178,6 +179,15 @@ const AUTOKEY_ENV = 'OS_CRYPTO_AUTOKEY'; * only WHICH file the file tier reads; an env key still answers first. */ export const KEY_FILE_CHANNEL_ENV = 'OS_INTERNAL_CRYPTO_KEY_FILE'; +/** + * The second internal channel, set beside {@link KEY_FILE_CHANNEL_ENV} only + * when {@link resolvePersistedKeyFile} answers `both-differ`: the legacy file, + * whose key the boot has not ruled out. The CLI parent cannot rule between the + * two files, because only the child opens the database; the child does, in + * {@link settleKeyFileChannel}, and then deletes this variable. Owned by the + * parent like every `OS_INTERNAL_*` channel, and not an operator setting. + */ +export const KEY_FILE_CANDIDATE_CHANNEL_ENV = 'OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE'; /** The data key's only legal length: AES-256 needs exactly 32 bytes. */ const DATA_KEY_BYTES = 32; @@ -473,14 +483,30 @@ const detectMode = (env: EnvMap): CryptoMode => { return 'development'; }; +/** + * A home directory spelled the way the CLI resolves `OS_HOME` for `os start` + * (`resolveStartHome` in `@objectstack/cli`): a leading `~` is the user's home + * directory, and the path is absolute. A shell expands `~` only when it is + * unquoted at the start of a word, so `OS_HOME="~/x"` reaches the process with + * the `~` intact; read literally, it named a directory called `~` under the + * working directory, and a key file there differed from the one `os start` + * used for the same value. + */ +const expandHome = (raw: string): string => { + const v = raw.trim(); + if (v.startsWith('~')) return resolvePath(homedir(), v.slice(1).replace(/^[/\\]/, '')); + return resolvePath(v); +}; + /** * Per-user persistent key location. Honours `OS_HOME` - * (legacy `OBJECTSTACK_HOME`) for projects that pin a non-default config dir. + * (legacy `OBJECTSTACK_HOME`) for projects that pin a non-default config dir, + * with a leading `~` expanded as {@link expandHome} says. */ const keyFilePath = (env: EnvMap): string => { + const pinned = env.OS_HOME?.trim() || env.OBJECTSTACK_HOME?.trim(); const home = - env.OS_HOME || - env.OBJECTSTACK_HOME || + (pinned ? expandHome(pinned) : undefined) || (env.HOME ? join(env.HOME, '.objectstack') : undefined) || join(homedir(), '.objectstack'); return join(home, 'dev-crypto-key'); @@ -597,6 +623,318 @@ export function resolvePersistedKeyFile( return { path: canonical, kind: 'canonical', canonical }; } +/** + * One stored `sys_secret` ciphertext, as {@link chooseKeyFileByTrialOpen} + * reads it: the row's own `(namespace, key)` and its sealed value. Nothing + * else of the row, and never the plaintext, leaves the trial. + */ +export interface StoredSecretSample { + namespace: string; + key: string; + ciphertext: string; +} + +/** + * Reads ONE stored ciphertext to judge a key by, or `undefined` when the + * database holds none. A failed read rejects: it is not an empty table. + */ +export type StoredSecretReader = () => Promise; + +/** The newest rows a reader looks through for one whose marker this provider knows. */ +const TRIAL_SAMPLE_PAGE = 10; + +/** + * The {@link StoredSecretReader} over a data engine: the NEWEST `sys_secret` + * row (by `created_at`) whose ciphertext records a derivation this provider + * knows, read as the system, outside any tenant. The newest row is the one the + * deployment sealed last, so the key that opens it is the key in use. + * + * A table that does not exist yet (a database that has never booted) holds no + * row. ⛔ Any other failed read rejects: it says nothing about the keys. + */ +export function storedSecretReaderOverEngine(engine: { + find(object: string, query?: Record): Promise; +}): StoredSecretReader { + return async () => { + let answer: unknown; + try { + answer = await engine.find('sys_secret', { + orderBy: [{ field: 'created_at', order: 'desc' }], + limit: TRIAL_SAMPLE_PAGE, + bypassTenantAudit: true, + context: { isSystem: true }, + }); + } catch (error) { + if (isMissingTableError(error, 'sys_secret')) return undefined; + throw error; + } + const rows = (Array.isArray(answer) ? answer : (answer as { data?: unknown[] } | null)?.data) ?? []; + for (const row of rows as Array>) { + if (ciphertextDerivationStatus(row?.ciphertext) === 'unknown') continue; + return { + namespace: String(row.namespace ?? ''), + key: String(row.key ?? ''), + ciphertext: row.ciphertext as string, + }; + } + return undefined; + }; +} + +/** + * Does `key` open `sample`? The trial asks only that: it opens the ciphertext + * under the derivation the ciphertext records, once per member of the closed + * scope set (a version-2 seal binds its producer's scope, and `sys_secret` + * records no producer), and throws the plaintext away. + * + * AES-GCM authenticates the key with the context, so a wrong key fails under + * every scope and the right key opens under exactly the scope that sealed the + * row. This decides WHICH KEY, never what a row means: no caller receives a + * value from it, so it guesses no producer for a read (ADR-0128 D3). + */ +async function keyOpensSample(key: Buffer, sample: StoredSecretSample): Promise { + const provider = new LocalCryptoProvider({ key }); + const handle: CryptoHandle = { + id: 'trial', + kmsKeyId: 'local:v1', + alg: 'aes-256-gcm', + version: 1, + ciphertext: sample.ciphertext, + }; + for (const scope of CRYPTO_CONTEXT_SCOPES) { + try { + await provider.decrypt(handle, { scope, namespace: sample.namespace, key: sample.key }); + return true; + } catch { + // Not this key under this scope; the next scope, or the verdict below. + } + } + return false; +} + +/** + * Which key file a boot reads, and why — {@link chooseKeyFileByTrialOpen}'s + * answer. + * + * - `single` — {@link resolvePersistedKeyFile} named one file; there was no + * second key to rule out. + * - `no-stored-secret` — two keys, but the database holds no ciphertext to + * judge them by: the canonical file, as before. + * - `opens` — the chosen file's key opens the newest stored ciphertext and the + * other's does not. + * - `opens-nothing` — neither key opens it (development: the canonical file, + * with a boot error; production refuses instead, see + * {@link DataKeyOpensNothingError}). + * - `not-judged` — the stored ciphertext could not be read: the canonical + * file, with a boot error naming the read. + */ +export interface KeyFileChoice { + path: string; + reason: 'single' | 'no-stored-secret' | 'opens' | 'opens-nothing' | 'not-judged'; + /** The file not chosen, when there were two. */ + other?: string; + /** What the boot prints, and at which level; absent for `single`. */ + notice?: { level: 'warn' | 'error'; message: string }; +} + +const OPENS_NOTHING_CONSEQUENCE = + `Every stored secret (encrypted settings, secret fields, datasource credentials) reads back as unset under either key.`; + +const opensNothingRemedy = (canonical: string): string => + `Fix: find the key these secrets were sealed under and put it at ${canonical} (keep a copy of both ` + + `files first), or set ${SECRET_KEY_ENV} to it.`; + +/** + * A production boot whose two candidate key files both fail to open the + * database's newest stored secret. The fail-loud contract this provider keeps + * (see "Fail-loud guarantee" above): production refuses to run under a key it + * cannot vouch for, because every secret would read as unset while the server + * looks healthy. + */ +export class DataKeyOpensNothingError extends Error { + constructor( + readonly canonical: string, + readonly legacy: string, + ) { + super( + `[LocalCryptoProvider] Refusing to start in production: neither data key opens this database's ` + + `stored secrets.\n` + + ` Tried:\n ${canonical}\n ${legacy}\n` + + ` ${OPENS_NOTHING_CONSEQUENCE}\n` + + ` ${opensNothingRemedy(canonical)}`, + ); + this.name = 'DataKeyOpensNothingError'; + } +} + +/** + * Rule between the two key files of a `both-differ` answer by the database + * itself: trial-open one stored `sys_secret` ciphertext (the newest one + * `readStoredSecret` returns) under each file's key, and take the file whose + * key opens it. The canonical file is tried first, so a ciphertext both keys + * open (impossible for two different AES keys, short of a forgery) keeps it. + * + * Every other answer of {@link resolvePersistedKeyFile} names one file and is + * returned as it is (`single`). With no stored ciphertext there is nothing to + * judge, and the canonical file stands (`no-stored-secret`). + * + * Neither key opens it: production throws {@link DataKeyOpensNothingError}; + * development and test keep the canonical file and return an `error` notice, + * the posture split of the provider's own fail-loud contract. ⛔ Never a + * silent choice: every answer between two files carries its notice. + */ +export async function chooseKeyFileByTrialOpen(opts: { + answer: PersistedKeyFile; + readStoredSecret: StoredSecretReader; + /** The posture the refusal is graded by. Defaults to `NODE_ENV`'s, as the provider reads it. */ + mode?: CryptoMode; + env?: EnvMap; +}): Promise { + const { answer } = opts; + if (answer.kind !== 'both-differ' || !answer.legacy) return { path: answer.path, reason: 'single' }; + const canonical = answer.canonical; + const legacy = answer.legacy; + const mode = opts.mode ?? detectMode(opts.env ?? processEnv()); + + let sample: StoredSecretSample | undefined; + try { + sample = await opts.readStoredSecret(); + } catch (error) { + const why = error instanceof Error ? error.message : String(error); + return { + path: canonical, + reason: 'not-judged', + other: legacy, + notice: { + level: 'error', + message: + `[LocalCryptoProvider] Data key: two different key files exist, ${canonical} and ${legacy}, and ` + + `the stored secrets could not be read to tell which one opens them (${why}). Using ${canonical}; ` + + `secrets sealed under ${legacy} will not decrypt. If they were saved under \`os start\`, keep a ` + + `copy of both files and move ${legacy} over ${canonical}.`, + }, + }; + } + + if (!sample) { + return { + path: canonical, + reason: 'no-stored-secret', + other: legacy, + notice: { + level: 'warn', + message: + `[LocalCryptoProvider] Data key: using ${canonical}. ${legacy} holds a different key; this ` + + `database has no stored secret yet to tell them apart, so the shared file stands.`, + }, + }; + } + + const canonicalKey = loadExistingKey(canonical); + const legacyKey = loadExistingKey(legacy); + if (canonicalKey && (await keyOpensSample(canonicalKey, sample))) { + return { + path: canonical, + reason: 'opens', + other: legacy, + notice: { + level: 'warn', + message: + `[LocalCryptoProvider] Data key: using ${canonical}, which opens this database's stored secrets. ` + + `${legacy} holds a different key that does not, and is not used.`, + }, + }; + } + if (legacyKey && (await keyOpensSample(legacyKey, sample))) { + return { + path: legacy, + reason: 'opens', + other: canonical, + notice: { + level: 'warn', + message: + `[LocalCryptoProvider] Data key: using ${legacy}, which opens this database's stored secrets ` + + `(an earlier \`os start\` kept it). ${canonical} holds a different key that does not, and is not ` + + `used. \`os dev\` and \`os start\` share ${canonical}: keep a copy of it, then move ${legacy} over ` + + `it to converge.`, + }, + }; + } + + if (mode === 'production') throw new DataKeyOpensNothingError(canonical, legacy); + return { + path: canonical, + reason: 'opens-nothing', + other: legacy, + notice: { + level: 'error', + message: + `[LocalCryptoProvider] Neither data key opens this database's stored secrets. Tried ${canonical} ` + + `(used) and ${legacy}. ${OPENS_NOTHING_CONSEQUENCE} ${opensNothingRemedy(canonical)}`, + }, + }; +} + +/** The settlement in flight per env map: concurrent callers share one trial. */ +const settlements = new WeakMap>(); + +/** + * Settle the CLI's key-file channel in the process that opened the database — + * the `serve` child, or a one-shot command after its boot. When the parent set + * {@link KEY_FILE_CANDIDATE_CHANNEL_ENV} beside {@link KEY_FILE_CHANNEL_ENV} + * (a `both-differ` answer), rule between the two files with + * {@link chooseKeyFileByTrialOpen}, point the channel at the chosen file, + * delete the candidate and print the notice once. Every provider built after + * this reads the chosen file; a provider built before it says so + * (`resolveDataKey`). + * + * `undefined` when there is nothing to settle: no candidate, or an earlier call + * already settled it. Concurrent callers share one trial. A production refusal + * ({@link DataKeyOpensNothingError}) rejects every caller of that trial, and + * both channels are left as they were. + */ +export function settleKeyFileChannel(opts: { + readStoredSecret: StoredSecretReader; + env?: EnvMap; + mode?: CryptoMode; +}): Promise { + const env = opts.env ?? processEnv(); + const pending = settlements.get(env); + if (pending) return pending; + const candidate = env[KEY_FILE_CANDIDATE_CHANNEL_ENV]?.trim(); + const current = env[KEY_FILE_CHANNEL_ENV]?.trim(); + if (!candidate || !current) return Promise.resolve(undefined); + const settlement = (async () => { + const choice = await chooseKeyFileByTrialOpen({ + answer: { path: current, kind: 'both-differ', canonical: current, legacy: candidate }, + readStoredSecret: opts.readStoredSecret, + mode: opts.mode, + env, + }); + env[KEY_FILE_CHANNEL_ENV] = choice.path; + delete env[KEY_FILE_CANDIDATE_CHANNEL_ENV]; + if (choice.notice) { + const print = (globalThis as { console?: Record void) | undefined> }).console?.[ + choice.notice.level + ]; + try { + print?.(choice.notice.message); + } catch { + /* exotic runtime without console — ignore */ + } + } + return choice; + })(); + settlements.set(env, settlement); + // Once settled the candidate is gone, so a later call has nothing to settle; + // after a refusal it is still there, and a later call judges again. + const clear = () => { + if (settlements.get(env) === settlement) settlements.delete(env); + }; + settlement.then(clear, clear); + return settlement; +} + const INVALID_KEY_MSG = (name: string): string => `[LocalCryptoProvider] ${name} is set but is not a 32-byte key (expected 64 hex chars or base64 of 32 bytes). ` + `Generate one with \`openssl rand -hex 32\`.`; @@ -666,6 +1004,16 @@ function resolveDataKey(opts: LocalCryptoProviderOptions): ResolvedKey { } const path = effectiveKeyFilePath(env); + const unsettled = env[KEY_FILE_CANDIDATE_CHANNEL_ENV]?.trim(); + if (unsettled) { + // A provider built before the boot ruled between the two key files + // (`settleKeyFileChannel`) reads the first one unexamined. Never silently. + warn( + `[LocalCryptoProvider] This provider was built before the boot chose its data key: it reads ${path}, ` + + `and ${unsettled} (a different key) was never tried against the stored secrets. Secrets sealed under ` + + `the other key will not decrypt through this provider.`, + ); + } if (mode === 'production') { // Honour a pre-existing, operator-provisioned key file first. @@ -876,6 +1224,47 @@ export class LocalCryptoProvider implements ICryptoProvider { } } +/** + * A {@link LocalCryptoProvider} built at its FIRST USE rather than now. + * + * A host has to hand a provider to consumers it composes before the database + * is open (a binder, a plugin's options), while the CLI's key-file channel is + * settled only once it is open ({@link settleKeyFileChannel}, in the settings + * service's `start()`). Built eagerly, such a provider would read the key file + * before the boot chose it. Built here, it reads the env as it stands at first + * use: every consumer of a kernel decrypts at `kernel:ready` or later, after + * every `start()`. + * + * `options` is read at that moment too, so an env snapshot it takes is the + * settled one. {@link DeferredLocalCryptoProvider.resolve} builds it at once, + * which is how a host keeps the constructor's production refusal at boot. A + * constructor that throws throws from every call, the same error each time. + */ +export interface DeferredLocalCryptoProvider extends ICryptoProvider { + /** The built provider's key source; reading it builds the provider. */ + readonly keySource: KeySource; + /** Build the provider now (once), or throw what its constructor throws. */ + resolve(): LocalCryptoProvider; +} + +export function deferredLocalCryptoProvider( + options: () => LocalCryptoProviderOptions = () => ({}), +): DeferredLocalCryptoProvider { + let built: LocalCryptoProvider | undefined; + const resolve = (): LocalCryptoProvider => (built ??= new LocalCryptoProvider(options())); + return { + get keySource() { + return resolve().keySource; + }, + resolve, + encrypt: async (plain, ctx) => resolve().encrypt(plain, ctx), + decrypt: async (handle, ctx) => resolve().decrypt(handle, ctx), + rotateKey: async (handle, ctx) => resolve().rotateKey(handle, ctx), + digest: (plain) => resolve().digest(plain), + keyedDigest: async (plain) => resolve().keyedDigest(plain), + }; +} + /** * @deprecated Renamed to {@link LocalCryptoProvider}. The old name implied an * "in-memory / ephemeral" key when the provider in fact persists its key diff --git a/packages/services/service-settings/src/settings-service-plugin.ts b/packages/services/service-settings/src/settings-service-plugin.ts index 628b86b4d9a..c963f2171fb 100644 --- a/packages/services/service-settings/src/settings-service-plugin.ts +++ b/packages/services/service-settings/src/settings-service-plugin.ts @@ -32,7 +32,11 @@ import { SettingsService } from './settings-service.js'; import type { ICryptoProvider } from '@objectstack/spec/contracts'; import type { SettingsAuditWriter, SettingsEngine, SettingsSecretStore } from './settings-service.types.js'; import type { CryptoAdapter } from './crypto-adapter.js'; -import { LocalCryptoProvider } from './local-crypto-provider.js'; +import { + LocalCryptoProvider, + settleKeyFileChannel, + storedSecretReaderOverEngine, +} from './local-crypto-provider.js'; import { buildConfigChangeAuditSink } from './config-change-audit.js'; import { registerLocalizationCurrencyLock, type CurrencyUsageEngine } from './localization-currency-lock.js'; import { USER_OBJECT, assertUserReferenceResolves, registeredLabel } from './actor-reference.js'; @@ -259,6 +263,26 @@ export class SettingsServicePlugin implements Plugin { async start(ctx: PluginContext): Promise { if (!this.service) return; + // The data key's file, where two candidates exist: ruled here, in Phase 2, + // because the database is open (this plugin starts after objectql) and no + // consumer has decrypted yet (they all read at `kernel:ready` or later, so + // every provider they build reads the chosen file). A no-op unless the CLI + // parent named a second candidate (`settleKeyFileChannel`). A production + // boot where neither key opens the stored secrets throws here, which stops + // the boot: the provider's fail-loud contract, at the one place that can + // judge it. + let settleEngine: IDataEngine | null = null; + try { + settleEngine = ctx.getService('objectql'); + } catch { + // No engine: no stored secret can be read, so the canonical file stands. + } + await settleKeyFileChannel({ + readStoredSecret: settleEngine + ? storedSecretReaderOverEngine(settleEngine as unknown as Parameters[0]) + : async () => undefined, + }); + ctx.hook('kernel:ready', async () => { // First, before the engine binds (its change event names every // registered namespace) and before any later `kernel:ready` reader runs. From e4d24a9c7e711a26245dcfefbcbb486c10dfefe1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:30:36 +0000 Subject: [PATCH 2/4] test(service-settings,cli): pin the data-key trial at start, the parent candidate channel, one-shot resolution and ~ expansion Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01WGufzhkoyLG9aL3MMfCgCW --- .../internal-crypto-key-file-channel.test.ts | 57 +++- .../src/utils/one-shot-settings.pin.test.ts | 75 ++++- .../src/local-crypto-provider.test.ts | 261 +++++++++++++++++- .../src/local-crypto-provider.ts | 21 +- .../src/settings-engine-bind-window.test.ts | 109 +++++++- 5 files changed, 498 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts b/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts index 74f8c44a065..9f364b9161b 100644 --- a/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts +++ b/packages/cli/src/utils/internal-crypto-key-file-channel.test.ts @@ -11,15 +11,17 @@ * boot, one settings secret) is `test/dev-start-shared-data-key.integration.test.ts`. */ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { randomBytes } from 'node:crypto'; -import { LocalCryptoProvider } from '@objectstack/service-settings'; +import { LocalCryptoProvider, resolvePersistedKeyFile, settleKeyFileChannel } from '@objectstack/service-settings'; import { applyCryptoKeyFileChannel } from './internal-crypto-key-file-channel.js'; +import { resolveStartHome } from './start-home.js'; const CHANNEL = 'OS_INTERNAL_CRYPTO_KEY_FILE'; +const CANDIDATE = 'OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE'; const ctx = { scope: 'settings', namespace: 'ai', key: 'openai_api_key' } as const; let home: string; @@ -102,16 +104,42 @@ describe('#29934 — the key file an earlier `os start` kept in the project', () expect(existsSync(canonicalFile())).toBe(false); }); - it('both files hold different keys: the canonical one is used, and the warning names both', async () => { + it('both files hold different keys: the child is handed both, and the notice names both', async () => { writeKey(legacyFile()); - const canonical = writeKey(canonicalFile()); - const sealed = await new LocalCryptoProvider({ key: canonical }).encrypt('sk-canonical', ctx); - + writeKey(canonicalFile()); for (const { child, notice } of [await startChild(), await devChild()]) { expect(child[CHANNEL]).toBe(canonicalFile()); - expect(notice).toContain(`${canonicalFile()} (used)`); - expect(notice).toContain(`${legacyFile()} (NOT used)`); - expect(await new LocalCryptoProvider({ env: child }).decrypt(sealed, ctx)).toBe('sk-canonical'); + // [#30042] Only the child opens the database, so the parent rules + // nothing: the second file travels on the candidate channel. + expect(child[CANDIDATE]).toBe(legacyFile()); + expect(notice).toContain(canonicalFile()); + expect(notice).toContain(legacyFile()); + } + }); + + it.each([ + ['the case-1 shape (a start-sealed project, a foreign home key)', 'legacy'], + ['the dev-then-start shape', 'canonical'], + ] as const)('%s: the child reads its secret back with the %s key', async (_shape, sealer) => { + const keys = { legacy: writeKey(legacyFile()), canonical: writeKey(canonicalFile()) }; + const handle = await new LocalCryptoProvider({ key: keys[sealer] }).encrypt('sk-30042', ctx); + const stored = async () => ({ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }); + for (const { child } of [await startChild(), await devChild()]) { + const quiet = vi.spyOn(console, 'warn').mockImplementation(() => {}); + // What the child's settings service does at start(), the database open. + const choice = await settleKeyFileChannel({ env: child, readStoredSecret: stored }); + quiet.mockRestore(); + expect(choice?.path).toBe(sealer === 'legacy' ? legacyFile() : canonicalFile()); + expect(child[CANDIDATE]).toBeUndefined(); + expect(await new LocalCryptoProvider({ env: child }).decrypt(handle, ctx)).toBe('sk-30042'); + } + }); + + it('an inherited candidate never survives a one-file answer', async () => { + writeKey(canonicalFile()); + const env = { ...operatorEnv, [CANDIDATE]: '/inherited/elsewhere' }; + for (const { child } of [await startChild(env), await devChild(env)]) { + expect(child[CANDIDATE]).toBeUndefined(); } }); @@ -131,10 +159,19 @@ describe('#29934 — an explicit env key keeps winning, unchanged', () => { ['OS_DEV_CRYPTO_KEY', randomBytes(32).toString('base64')], ])('%s set: no file is named and an inherited channel is removed', async (name, value) => { writeKey(legacyFile()); - const env = { ...operatorEnv, [name]: value, [CHANNEL]: '/inherited/elsewhere' }; + const env = { ...operatorEnv, [name]: value, [CHANNEL]: '/inherited/elsewhere', [CANDIDATE]: '/inherited/too' }; for (const { child, notice } of [await startChild(env), await devChild(env)]) { expect(child[CHANNEL]).toBeUndefined(); + expect(child[CANDIDATE]).toBeUndefined(); expect(notice).toBeUndefined(); } }); }); + +describe('#30042 — a leading `~` in OS_HOME names one directory for the key file and for `os start`', () => { + it.each(['~/pinned-home', '~'])('OS_HOME=%s', (value) => { + const env = { ...operatorEnv, OS_HOME: value }; + const startHome = resolveStartHome(undefined, { hasProjectConfig: true, cwd: project, env }); + expect(resolvePersistedKeyFile({ env })?.canonical).toBe(join(startHome, 'dev-crypto-key')); + }); +}); diff --git a/packages/cli/src/utils/one-shot-settings.pin.test.ts b/packages/cli/src/utils/one-shot-settings.pin.test.ts index cf13aea53f7..0f18d040d88 100644 --- a/packages/cli/src/utils/one-shot-settings.pin.test.ts +++ b/packages/cli/src/utils/one-shot-settings.pin.test.ts @@ -34,12 +34,13 @@ * `schema-migrate.one-shot-family.integration.test.ts`. */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { randomBytes } from 'node:crypto'; import { tmpdir } from 'node:os'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { LocalCryptoProvider } from '@objectstack/service-settings'; +import { LocalCryptoProvider, settleKeyFileChannel } from '@objectstack/service-settings'; // The one code/prose separator, typed by the hand-written `.d.mts` beside it. import { maskComments } from '../../../../scripts/js-comment-mask.mjs'; import { oneShotSettingsPlugin, refusingCryptoProvider, resolveExistingDataKey } from './one-shot-settings.js'; @@ -104,6 +105,7 @@ describe('[#21471] the settings service is composed in one place in this package const KEY_ENV = [ 'NODE_ENV', 'OS_SECRET_KEY', 'OS_DEV_CRYPTO_KEY', 'OBJECTSTACK_DEV_CRYPTO_KEY', 'OS_HOME', 'OBJECTSTACK_HOME', 'OS_CRYPTO_AUTOKEY', + 'OS_INTERNAL_CRYPTO_KEY_FILE', 'OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE', ] as const; const KEY_HEX = '606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f'; const savedEnv: Record = {}; @@ -202,3 +204,70 @@ describe('[#21471] the one-shot helper never mints a data key', () => { await expect(provider.keyedDigest('x')).rejects.toThrow(/the reason/); }); }); + +// ── [#30042] The file a one-shot reads is the one `os dev` / `os start` read ── + +describe('[#30042] one-shot commands take the same key file, by the same resolution and trial', () => { + let project: string; + let userHome: string; + beforeEach(() => { + project = mkdtempSync(join(tmpdir(), 'os-30042-project-')); + userHome = mkdtempSync(join(tmpdir(), 'os-30042-user-')); + writeFileSync(join(project, 'objectstack.config.ts'), 'export default {};\n'); + }); + afterEach(() => { + rmSync(project, { recursive: true, force: true }); + rmSync(userHome, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + const canonicalFile = () => join(userHome, '.objectstack', 'dev-crypto-key'); + const legacyFile = () => join(project, '.objectstack', 'dev-crypto-key'); + const writeKey = (file: string): Buffer => { + const key = randomBytes(32); + mkdirSync(join(file, '..'), { recursive: true }); + writeFileSync(file, key.toString('base64'), { mode: 0o600 }); + return key; + }; + const ctx = { scope: 'settings', namespace: 'ai', key: 'openai_api_key' } as const; + /** A one-shot's own environment: no key and no home pin, `HOME` is this test's. */ + const oneShotEnv = (): NodeJS.ProcessEnv => ({ HOME: userHome, NODE_ENV: 'production' }); + + it('only the project file an earlier `os start` kept holds a key: the one-shot reads it', async () => { + const legacy = writeKey(legacyFile()); + const handle = await new LocalCryptoProvider({ key: legacy }).encrypt('sk-legacy', ctx); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const env = oneShotEnv(); + const key = await resolveExistingDataKey({ cwd: project, env }); + expect(await key.provider!.decrypt(handle, ctx)).toBe('sk-legacy'); + expect(existsSync(canonicalFile())).toBe(false); + }); + + it('two different keys: the provider is the one the boot\'s trial chose', async () => { + writeKey(canonicalFile()); + const legacy = writeKey(legacyFile()); + const handle = await new LocalCryptoProvider({ key: legacy }).encrypt('sk-start', ctx); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const env = oneShotEnv(); + const key = await resolveExistingDataKey({ cwd: project, env }); + expect(key.unavailable).toBeNull(); + expect(env.OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE).toBe(legacyFile()); + // What the boot's settings service does at start(), the database open. + await settleKeyFileChannel({ + env, + readStoredSecret: async () => ({ namespace: ctx.namespace, key: ctx.key, ciphertext: handle.ciphertext }), + }); + expect(await key.provider!.decrypt(handle, ctx)).toBe('sk-start'); + expect(key.provider!.keySource).toBe('file'); + }); + + it('two different keys with no trial yet: nothing has been read, and nothing is minted', async () => { + writeKey(canonicalFile()); + writeKey(legacyFile()); + const env = oneShotEnv(); + const before = readdirSync(join(userHome, '.objectstack')); + const key = await resolveExistingDataKey({ cwd: project, env }); + expect(key.provider).not.toBeNull(); + expect(readdirSync(join(userHome, '.objectstack'))).toEqual(before); + }); +}); diff --git a/packages/services/service-settings/src/local-crypto-provider.test.ts b/packages/services/service-settings/src/local-crypto-provider.test.ts index 13667a0bb28..f843e4aabec 100644 --- a/packages/services/service-settings/src/local-crypto-provider.test.ts +++ b/packages/services/service-settings/src/local-crypto-provider.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { createHash, createHmac, randomBytes } from 'node:crypto'; import { @@ -21,6 +21,14 @@ import { ciphertextDerivationStatus, resolvePersistedKeyFile, KEY_FILE_CHANNEL_ENV, + KEY_FILE_CANDIDATE_CHANNEL_ENV, + chooseKeyFileByTrialOpen, + settleKeyFileChannel, + storedSecretReaderOverEngine, + deferredLocalCryptoProvider, + DataKeyOpensNothingError, + type PersistedKeyFile, + type StoredSecretSample, } from './local-crypto-provider.js'; const ctx: CryptoContext = { scope: 'settings', namespace: 'mail', key: 'api_key' }; @@ -231,6 +239,257 @@ describe('#29934 — the key file the CLI names, and the one resolution behind i }); }); +describe('#30042 — two key files: the boot takes the one that opens a stored secret', () => { + let home: string; + let other: string; + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'os-30042-home-')); + other = mkdtempSync(join(tmpdir(), 'os-30042-other-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + rmSync(other, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + const canonical = () => join(home, '.objectstack', 'dev-crypto-key'); + const legacy = () => join(other, 'dev-crypto-key'); + const writeKey = (file: string): Buffer => { + const key = randomBytes(32); + mkdirSync(join(file, '..'), { recursive: true }); + writeFileSync(file, key.toString('base64'), { mode: 0o600 }); + return key; + }; + const bothDiffer = (): PersistedKeyFile => ({ + path: canonical(), + kind: 'both-differ', + canonical: canonical(), + legacy: legacy(), + }); + /** A stored row as the reader returns it: sealed under `key` by the producer of `scope`. */ + const sealed = async (key: Buffer, scope: CryptoContextScope = 'settings'): Promise => { + const h = await new LocalCryptoProvider({ key }).encrypt('sk-30042', { scope, namespace: 'ai', key: 'openai_api_key' }); + return { namespace: 'ai', key: 'openai_api_key', ciphertext: h.ciphertext }; + }; + const reader = (sample: StoredSecretSample | undefined) => async () => sample; + + it('the case-1 shape: a start-sealed database with a foreign home key reads with the legacy key', async () => { + writeKey(canonical()); // `os dev` in another project minted this one + const startKey = writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(startKey)), + mode: 'production', + }); + expect(choice).toMatchObject({ path: legacy(), reason: 'opens', other: canonical() }); + expect(choice.notice?.level).toBe('warn'); + expect(choice.notice?.message).toContain(legacy()); + expect(choice.notice?.message).toContain(canonical()); + }); + + it('the dev-then-start shape: a database the canonical key sealed keeps the canonical key', async () => { + const devKey = writeKey(canonical()); + writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(devKey)), + mode: 'production', + }); + expect(choice).toMatchObject({ path: canonical(), reason: 'opens', other: legacy() }); + }); + + it.each(CRYPTO_CONTEXT_SCOPES)('a row sealed by the %s producer is judged too (sys_secret records no scope)', async (scope) => { + writeKey(canonical()); + const startKey = writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(startKey, scope)), + mode: 'production', + }); + expect(choice.path).toBe(legacy()); + }); + + it('a key that opens nothing refuses a production boot, naming both paths', async () => { + writeKey(canonical()); + writeKey(legacy()); + const elsewhere = await sealed(randomBytes(32)); + const refusal = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(elsewhere), + mode: 'production', + }).catch((e: unknown) => e); + expect(refusal).toBeInstanceOf(DataKeyOpensNothingError); + const message = (refusal as Error).message; + // serve's secret-field wiring rethrows exactly this phrase as a boot refusal. + expect(message).toContain('Refusing to start in production'); + expect(message).toContain(canonical()); + expect(message).toContain(legacy()); + }); + + it.each(['development', 'test'] as const)('%s: a key that opens nothing is a boot ERROR naming both paths, on the canonical file', async (mode) => { + writeKey(canonical()); + writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: reader(await sealed(randomBytes(32))), + mode, + }); + expect(choice).toMatchObject({ path: canonical(), reason: 'opens-nothing' }); + expect(choice.notice?.level).toBe('error'); + expect(choice.notice?.message).toContain(canonical()); + expect(choice.notice?.message).toContain(legacy()); + }); + + it('no stored secret: nothing to judge, the canonical file stands', async () => { + writeKey(canonical()); + writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ answer: bothDiffer(), readStoredSecret: reader(undefined), mode: 'production' }); + expect(choice).toMatchObject({ path: canonical(), reason: 'no-stored-secret' }); + }); + + it('a read that fails is no verdict: the canonical file, with a boot error naming the read', async () => { + writeKey(canonical()); + writeKey(legacy()); + const choice = await chooseKeyFileByTrialOpen({ + answer: bothDiffer(), + readStoredSecret: async () => { + throw new Error('connection reset'); + }, + mode: 'production', + }); + expect(choice).toMatchObject({ path: canonical(), reason: 'not-judged' }); + expect(choice.notice?.level).toBe('error'); + expect(choice.notice?.message).toContain('connection reset'); + }); + + it('every one-file answer passes through untried', async () => { + const read = vi.fn(async () => undefined); + for (const answer of [ + { path: canonical(), kind: 'canonical', canonical: canonical() }, + { path: legacy(), kind: 'legacy-only', canonical: canonical(), legacy: legacy() }, + ] as PersistedKeyFile[]) { + expect(await chooseKeyFileByTrialOpen({ answer, readStoredSecret: read })).toEqual({ path: answer.path, reason: 'single' }); + } + expect(read).not.toHaveBeenCalled(); + }); + + it('settleKeyFileChannel points the channel at the chosen file, drops the candidate, and says so once', async () => { + writeKey(canonical()); + const startKey = writeKey(legacy()); + const sample = await sealed(startKey); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const env: Record = { + NODE_ENV: 'production', + [KEY_FILE_CHANNEL_ENV]: canonical(), + [KEY_FILE_CANDIDATE_CHANNEL_ENV]: legacy(), + }; + const read = vi.fn(reader(sample)); + const [a, b] = await Promise.all([ + settleKeyFileChannel({ env, readStoredSecret: read }), + settleKeyFileChannel({ env, readStoredSecret: read }), + ]); + expect(a?.path).toBe(legacy()); + expect(b).toBe(a); + expect(read).toHaveBeenCalledTimes(1); + expect(env[KEY_FILE_CHANNEL_ENV]).toBe(legacy()); + expect(env[KEY_FILE_CANDIDATE_CHANNEL_ENV]).toBeUndefined(); + expect(warn.mock.calls.filter(([m]) => String(m).includes(legacy()))).toHaveLength(1); + expect(await settleKeyFileChannel({ env, readStoredSecret: read })).toBeUndefined(); + // A provider built after the settlement opens the stored secret. + const handle = { id: 'x', kmsKeyId: 'local:v1', alg: 'aes-256-gcm', version: 1, ciphertext: sample.ciphertext }; + expect(await new LocalCryptoProvider({ env }).decrypt(handle, { scope: 'settings', namespace: 'ai', key: 'openai_api_key' })).toBe('sk-30042'); + }); + + it('a production refusal leaves both channels as they were', async () => { + writeKey(canonical()); + writeKey(legacy()); + const env: Record = { + NODE_ENV: 'production', + [KEY_FILE_CHANNEL_ENV]: canonical(), + [KEY_FILE_CANDIDATE_CHANNEL_ENV]: legacy(), + }; + await expect( + settleKeyFileChannel({ env, readStoredSecret: reader(await sealed(randomBytes(32))) }), + ).rejects.toBeInstanceOf(DataKeyOpensNothingError); + expect(env[KEY_FILE_CHANNEL_ENV]).toBe(canonical()); + expect(env[KEY_FILE_CANDIDATE_CHANNEL_ENV]).toBe(legacy()); + }); + + it('development: neither key opens it → console.error, never a debug line', async () => { + writeKey(canonical()); + writeKey(legacy()); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + const env: Record = { + NODE_ENV: 'development', + [KEY_FILE_CHANNEL_ENV]: canonical(), + [KEY_FILE_CANDIDATE_CHANNEL_ENV]: legacy(), + }; + const choice = await settleKeyFileChannel({ env, readStoredSecret: reader(await sealed(randomBytes(32))) }); + expect(choice?.reason).toBe('opens-nothing'); + expect(error).toHaveBeenCalledTimes(1); + expect(String(error.mock.calls[0]?.[0])).toContain(legacy()); + }); + + it('nothing to settle without a candidate', async () => { + const read = vi.fn(reader(undefined)); + expect(await settleKeyFileChannel({ env: { [KEY_FILE_CHANNEL_ENV]: canonical() }, readStoredSecret: read })).toBeUndefined(); + expect(read).not.toHaveBeenCalled(); + }); + + it('a provider built before the settlement says so', () => { + writeKey(canonical()); + writeKey(legacy()); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + new LocalCryptoProvider({ + env: { NODE_ENV: 'production', [KEY_FILE_CHANNEL_ENV]: canonical(), [KEY_FILE_CANDIDATE_CHANNEL_ENV]: legacy() }, + }); + expect(warn.mock.calls.some(([m]) => String(m).includes(legacy()) && String(m).includes('never tried'))).toBe(true); + }); + + it('deferredLocalCryptoProvider reads its options at first use, not at construction', async () => { + const first = writeKey(canonical()); + const env: Record = { NODE_ENV: 'production', [KEY_FILE_CHANNEL_ENV]: '/nowhere/yet' }; + const deferred = deferredLocalCryptoProvider(() => ({ env: { ...env } })); + env[KEY_FILE_CHANNEL_ENV] = canonical(); + const h = await new LocalCryptoProvider({ key: first }).encrypt('v', ctx); + expect(await deferred.decrypt(h, ctx)).toBe('v'); + expect(deferred.keySource).toBe('file'); + expect(deferred.resolve()).toBe(deferred.resolve()); + }); +}); + +describe('#30042 — the reader over the engine', () => { + const v2 = 'v2:' + randomBytes(40).toString('base64'); + it('reads the newest rows as the system and skips a marker the provider does not know', async () => { + const find = vi.fn(async () => [ + { namespace: 'x', key: 'y', ciphertext: 'v9:abc' }, + { namespace: 'ai', key: 'openai_api_key', ciphertext: v2 }, + ]); + expect(await storedSecretReaderOverEngine({ find })()).toEqual({ namespace: 'ai', key: 'openai_api_key', ciphertext: v2 }); + expect(find).toHaveBeenCalledWith('sys_secret', expect.objectContaining({ + orderBy: [{ field: 'created_at', order: 'desc' }], + context: { isSystem: true }, + })); + }); + + it('a table that does not exist holds no row; any other failure rejects', async () => { + const missing = storedSecretReaderOverEngine({ find: async () => { throw new Error('no such table: sys_secret'); } }); + expect(await missing()).toBeUndefined(); + const broken = storedSecretReaderOverEngine({ find: async () => { throw new Error('connection reset'); } }); + await expect(broken()).rejects.toThrow('connection reset'); + }); +}); + +describe('#30042 — a leading `~` in OS_HOME is the home directory, as the CLI reads it', () => { + it.each([ + ['~/pinned', join(homedir(), 'pinned', 'dev-crypto-key')], + ['~', join(homedir(), 'dev-crypto-key')], + ])('OS_HOME=%s', (value, file) => { + expect(resolvePersistedKeyFile({ env: { OS_HOME: value } })?.canonical).toBe(file); + expect(resolvePersistedKeyFile({ env: { OBJECTSTACK_HOME: value } })?.canonical).toBe(file); + }); +}); + describe('LocalCryptoProvider — crypto semantics', () => { it('AAD binding rejects ciphertexts swapped across (namespace,key)', async () => { const p = new LocalCryptoProvider({ key: randomBytes(32) }); diff --git a/packages/services/service-settings/src/local-crypto-provider.ts b/packages/services/service-settings/src/local-crypto-provider.ts index 0735550bbcb..ff1d4147256 100644 --- a/packages/services/service-settings/src/local-crypto-provider.ts +++ b/packages/services/service-settings/src/local-crypto-provider.ts @@ -913,16 +913,8 @@ export function settleKeyFileChannel(opts: { }); env[KEY_FILE_CHANNEL_ENV] = choice.path; delete env[KEY_FILE_CANDIDATE_CHANNEL_ENV]; - if (choice.notice) { - const print = (globalThis as { console?: Record void) | undefined> }).console?.[ - choice.notice.level - ]; - try { - print?.(choice.notice.message); - } catch { - /* exotic runtime without console — ignore */ - } - } + if (choice.notice?.level === 'warn') warn(choice.notice.message); + else if (choice.notice) bootError(choice.notice.message); return choice; })(); settlements.set(env, settlement); @@ -962,6 +954,15 @@ const warn = (msg: string): void => { } }; +/** A boot error the operator must see: `console.error`, never a debug line. */ +const bootError = (msg: string): void => { + try { + (globalThis as { console?: { error?: (m: string) => void } }).console?.error?.(msg); + } catch { + /* exotic runtime without console — ignore */ + } +}; + const legacyDeprecationWarned = { value: false }; function resolveDataKey(opts: LocalCryptoProviderOptions): ResolvedKey { diff --git a/packages/services/service-settings/src/settings-engine-bind-window.test.ts b/packages/services/service-settings/src/settings-engine-bind-window.test.ts index 5a3f10ba664..ebf19a66e6a 100644 --- a/packages/services/service-settings/src/settings-engine-bind-window.test.ts +++ b/packages/services/service-settings/src/settings-engine-bind-window.test.ts @@ -51,7 +51,11 @@ * on the new one — and asserts the second. */ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomBytes } from 'node:crypto'; import { LiteKernel } from '@objectstack/core'; import type { Plugin, PluginContext } from '@objectstack/core'; import { ObjectQL } from '@objectstack/objectql'; @@ -60,6 +64,12 @@ import type { SettingsManifest } from '@objectstack/spec/system'; import { SettingsService } from './settings-service.js'; import { SettingsServicePlugin, wrapEngineAsSettingsEngine } from './settings-service-plugin.js'; import { SettingsEngineNotBoundError } from './settings-service.types.js'; +import { + DataKeyOpensNothingError, + KEY_FILE_CANDIDATE_CHANNEL_ENV, + KEY_FILE_CHANNEL_ENV, + LocalCryptoProvider, +} from './local-crypto-provider.js'; const OWNER_PACKAGE = 'com.objectstack.test.settings-engine-bind-window'; @@ -400,3 +410,100 @@ describe('engine-less callers outside the window observe nothing new', () => { expect(resolved.value).toBe('now-fine'); }); }); + +// --------------------------------------------------------------------------- +// #30042 — the data key's file is chosen in start(), before every +// `kernel:ready` reader +// --------------------------------------------------------------------------- + +/** Records, at `kernel:ready`, which key file the process env names by then. */ +class KeyFileAtReadyProbe implements Plugin { + name = 'com.objectstack.test.key-file-at-ready'; + version = '0.0.0'; + type = 'standard' as const; + channelAtReady: string | undefined = 'never-ran'; + candidateAtReady: string | undefined = 'never-ran'; + init = async (ctx: PluginContext) => { + ctx.hook('kernel:ready', async () => { + this.channelAtReady = process.env[KEY_FILE_CHANNEL_ENV]; + this.candidateAtReady = process.env[KEY_FILE_CANDIDATE_CHANNEL_ENV]; + }); + }; +} + +describe('#30042 — the settings service chooses the data key file at start(), with the database open', () => { + const touched = [KEY_FILE_CHANNEL_ENV, KEY_FILE_CANDIDATE_CHANNEL_ENV, 'NODE_ENV'] as const; + const saved = Object.fromEntries(touched.map((k) => [k, process.env[k]])); + let dir: string | undefined; + afterEach(() => { + for (const k of touched) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + vi.restoreAllMocks(); + }); + + /** Two key files, one stored row sealed under `sealedUnder`, and the channels a `both-differ` parent sets. */ + async function arrange(sealedUnder: 'legacy' | 'elsewhere') { + dir = mkdtempSync(join(tmpdir(), 'os-30042-plugin-')); + const canonical = join(dir, 'home', 'dev-crypto-key'); + const legacy = join(dir, 'project', 'dev-crypto-key'); + const keys: Record = {}; + for (const [name, file] of [['canonical', canonical], ['legacy', legacy]] as const) { + keys[name] = randomBytes(32); + mkdirSync(join(file, '..'), { recursive: true }); + writeFileSync(file, keys[name].toString('base64'), { mode: 0o600 }); + } + const sealKey = sealedUnder === 'legacy' ? keys.legacy : randomBytes(32); + const handle = await new LocalCryptoProvider({ key: sealKey }).encrypt('sk-start', { + scope: 'settings', + namespace: 'ai', + key: 'openai_api_key', + }); + const { driver, rowsOf } = makeMemoryDriver(); + rowsOf('sys_secret').set(handle.id, { + id: handle.id, + namespace: 'ai', + key: 'openai_api_key', + ciphertext: handle.ciphertext, + kms_key_id: handle.kmsKeyId, + alg: handle.alg, + version: handle.version, + created_at: new Date().toISOString(), + }); + const engine = new ObjectQL(); + engine.registerDriver(driver, true); + await engine.init(); + for (const o of [SysSetting, SysPlatformSetting, SysSecret, SysSettingAudit, SysPlatformSettingAudit]) { + engine.registry.registerObject(o as any, OWNER_PACKAGE); + } + process.env[KEY_FILE_CHANNEL_ENV] = canonical; + process.env[KEY_FILE_CANDIDATE_CHANNEL_ENV] = legacy; + const probe = new KeyFileAtReadyProbe(); + const kernel = new LiteKernel({ logger: { level: 'error' } as never }); + kernel.use(new EnginePlugin(engine)); + kernel.use(new SettingsServicePlugin({ registerRoutes: false, manifests: [], actionHandlers: {} })); + kernel.use(probe); + return { kernel, probe, canonical, legacy }; + } + + it('a database sealed under the legacy key: every kernel:ready reader sees the legacy file chosen', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const { kernel, probe, legacy } = await arrange('legacy'); + await kernel.bootstrap(); + expect(probe.channelAtReady).toBe(legacy); + expect(probe.candidateAtReady).toBeUndefined(); + }); + + it('production: a key that opens nothing stops the boot, naming both files', async () => { + process.env.NODE_ENV = 'production'; + const { kernel, probe, canonical, legacy } = await arrange('elsewhere'); + const refusal = await kernel.bootstrap().then(() => undefined, (e: unknown) => e); + expect(refusal).toBeInstanceOf(DataKeyOpensNothingError); + expect((refusal as Error).message).toContain(canonical); + expect((refusal as Error).message).toContain(legacy); + expect(probe.channelAtReady).toBe('never-ran'); + }); +}); From 014519dfbc40e0b54438c8d0850270cb3d9e0d47 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:32:13 +0000 Subject: [PATCH 3/4] test(cli): pin the two-key-file os start boot at the public door Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01WGufzhkoyLG9aL3MMfCgCW --- ...-start-shared-data-key.integration.test.ts | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/dev-start-shared-data-key.integration.test.ts b/packages/cli/test/dev-start-shared-data-key.integration.test.ts index 3aece44132b..a3ad09e1286 100644 --- a/packages/cli/test/dev-start-shared-data-key.integration.test.ts +++ b/packages/cli/test/dev-start-shared-data-key.integration.test.ts @@ -1,5 +1,5 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. -// test-budget: 20 — spawns the real producer once per single-cause fixture: two boots in one beforeAll, `os dev` then `os start` over one project; first reading 11.6 s on a shared local box (no CI reading yet) x 1.25, rounded up +// test-budget: 40 — spawns the real producer once per single-cause fixture: `os dev` then `os start` over one project, then two `os start` boots over that project's two key files (one reads back, one refuses); reading 24.3 s for the four boots on a shared local box (no CI reading yet) x 1.25, rounded up to the next 10 s /** * #29934 — a secret saved under `os dev` reads back under `os start` over the @@ -37,7 +37,8 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { spawn, type ChildProcess } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { randomBytes } from 'node:crypto'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -124,6 +125,7 @@ function boot(what: string, args: string[]): Promise { OS_DEV_CRYPTO_KEY: undefined, OBJECTSTACK_DEV_CRYPTO_KEY: undefined, OS_INTERNAL_CRYPTO_KEY_FILE: undefined, + OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE: undefined, OS_HOME: undefined, OBJECTSTACK_HOME: undefined, OS_DATABASE_URL: undefined, @@ -288,3 +290,96 @@ describe('#29934 — `os dev` and `os start` share one data key for one project' expect(existsSync(join(dir, '.objectstack', 'dev-crypto-key'))).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// #30042 — two key files: `os start` reads with the one that opens the secrets +// --------------------------------------------------------------------------- + +/** + * The card's case 1, at the public door, on the project the boots above left: + * its database holds the secret sealed under `$HOME/.objectstack/dev-crypto-key`. + * Moving that file to `/.objectstack/dev-crypto-key` and writing a + * DIFFERENT key at the home path is exactly a project that only ever ran + * `os start` before #29934 (its key in the project), on a machine whose home + * holds a key from `os dev` in another project. `os start` must read the secret + * back with the project's key. Then a third key in the project's place, so + * neither file opens anything: `os start` (a production boot) must refuse, + * naming both files. + */ +interface TwoKeyReading { + get?: { status: number; body: any }; + chosen?: string; + refusal?: string; + transcript?: string; + error?: Error; +} +let twoKeys: TwoKeyReading = {}; + +const writeForeignKey = (file: string) => { + mkdirSync(join(file, '..'), { recursive: true }); + writeFileSync(file, randomBytes(32).toString('base64'), { mode: 0o600 }); +}; + +describe('#30042 — two key files: the boot takes the one that opens the stored secrets', () => { + const homeKey = () => join(home, '.objectstack', 'dev-crypto-key'); + const projectKey = () => join(dir, '.objectstack', 'dev-crypto-key'); + + beforeAll(async () => { + let live: Live | undefined; + try { + recorded(); + renameSync(homeKey(), projectKey()); + writeForeignKey(homeKey()); + live = await boot('os start (two keys)', ['start', '--artifact', './dist/objectstack.json', '--port', KERNEL_ASSIGNED_PORT]); + const token = await signIn(live); + const get = await http(live, 'GET', '/api/settings/ai', token); + const transcript = live.output().slice(-4000); + await stopGroup(live.child); + live = undefined; + + // Neither file opens the stored secret now. + writeForeignKey(projectKey()); + const refusal = await boot('os start (no key opens)', [ + 'start', '--artifact', './dist/objectstack.json', '--port', KERNEL_ASSIGNED_PORT, + ]).then( + async (l) => { + live = l; + return `booted: ${l.output().slice(-2000)}`; + }, + (e: unknown) => (e instanceof Error ? e.message : String(e)), + ); + twoKeys = { get, chosen: transcript, refusal, transcript }; + } catch (error) { + twoKeys = { error: error instanceof Error ? error : new Error(String(error)) }; + } finally { + if (live) await stopGroup(live.child); + } + }, 2 * BOOT_TIMEOUT_MS + 60_000); + + function read(): Required> { + if (twoKeys.error) throw twoKeys.error; + return twoKeys as Required>; + } + + it('a start-sealed project beside a foreign home key reads its secret back', () => { + const { get, transcript } = read(); + expect(get.status, `${JSON.stringify(get.body).slice(0, 2000)}\n${transcript}`).toBe(200); + const entry = aiKey(get.body); + const why = `openai_api_key = ${JSON.stringify(entry)}\n${transcript}`; + expect(entry?.source, why).toBe('global'); + expect(entry?.value, why).not.toBeNull(); + }); + + it('the boot says which file it took and why', () => { + const { chosen } = read(); + expect(chosen).toContain(`using ${projectKey()}`); + }); + + it('a production boot whose two keys open nothing refuses, naming both files', () => { + const { refusal } = read(); + expect(refusal).not.toMatch(/^booted:/); + expect(refusal).toContain('Refusing to start in production'); + expect(refusal).toContain(homeKey()); + expect(refusal).toContain(projectKey()); + }); +}); From 42f8d6a61ce9b500e3370ad6761f1d229f3d9274 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:34:45 +0000 Subject: [PATCH 4/4] chore(changeset): declare the two-key-file boot choice and its production refusal Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01WGufzhkoyLG9aL3MMfCgCW --- ...cks-the-key-file-that-opens-its-secrets.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changeset/30042-boot-picks-the-key-file-that-opens-its-secrets.md diff --git a/.changeset/30042-boot-picks-the-key-file-that-opens-its-secrets.md b/.changeset/30042-boot-picks-the-key-file-that-opens-its-secrets.md new file mode 100644 index 00000000000..53a348939ad --- /dev/null +++ b/.changeset/30042-boot-picks-the-key-file-that-opens-its-secrets.md @@ -0,0 +1,20 @@ +--- +'@objectstack/cli': patch +'@objectstack/service-settings': patch +--- + +fix(cli): when two data-key files hold different keys, `os start` and `os dev` read with the one that opens the project's stored secrets, or a production boot refuses + +Clause-②: yes (narrowing) — a production `os start` / `os serve` boot whose two candidate key files (the shared `$HOME/.objectstack/dev-crypto-key` or `$OS_HOME`'s, and the project's `.objectstack/dev-crypto-key`) both fail to open the database's newest stored secret now refuses to start, naming both files; it used to start and serve every stored secret as unset. The widening half: `@objectstack/service-settings` gains new exports (`chooseKeyFileByTrialOpen`, `settleKeyFileChannel`, `storedSecretReaderOverEngine`, `deferredLocalCryptoProvider`, `DataKeyOpensNothingError`, `KEY_FILE_CANDIDATE_CHANNEL_ENV` and their types). +Metadata-protocol: none + + + +**BREAKING** boot narrowing (`@objectstack/cli`, `@objectstack/service-settings`): a production boot whose candidate data keys open none of the stored secrets refuses to start. + +- **`@objectstack/cli`.** When the shared key file and the project's own key file (the one an earlier `os start` minted) hold different keys, the CLI no longer picks one. It hands the `serve` child both, and the child decides once the database is open: it opens the newest stored `sys_secret` value under each key and reads with the key that opens it, and the boot says which file it took. A project that only ever ran `os start`, on a machine whose home holds a key from `os dev` in another project, reads its secrets back again. With no stored secret yet, the shared file is used, as before. When neither key opens the stored secrets, a production boot (`os start`, `os serve`) refuses to start and names both files; a development boot (`os dev`) starts and prints a boot error naming both files. The one-shot commands (`os migrate`, `os secret rewrap`, `os secret orphans`, `os storage orphans`) now read the same file by the same rules: the project's own file when it alone holds a key, and the key that opens a stored secret when both do. +- **`@objectstack/service-settings`.** The settings service makes that choice in its `start()`, before any reader decrypts, when the CLI names a second candidate on the internal `OS_INTERNAL_CRYPTO_KEY_FILE_CANDIDATE` channel. A leading `~` in `OS_HOME` (or `OBJECTSTACK_HOME`) is now the home directory when the provider finds its key file, as `os start` already read it; it used to name a directory called `~` under the working directory. + +**The remedy.** If a production boot refuses with "neither data key opens this database's stored secrets", find the key those secrets were sealed under (a backup of either file, or the `OS_SECRET_KEY` the deployment once ran with), keep a copy of both files, and put that key at the shared file the message names, or set `OS_SECRET_KEY` to it. + +Supersedes, in the same release: the note for #29934 says that when the shared file and the project's file hold different keys, the shared one is used and a warning names both paths. After this change the boot uses whichever of the two opens the stored secrets, and refuses in production when neither does.