From 5d5bd60029d7b25d08ab886e0b6a3cf7e18f5ad4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:00:16 +0000 Subject: [PATCH 1/7] feat(metadata-core): one comparison of an installed package version with the incoming one The package upgrade plan lists objects and fields added and removed, type changes with their drift category, renames a field declares with `renamedFrom`, suspected renames nothing declares, and seed datasets added, removed and changed with their modes. A declared rename whose type changes is also a type change, judged on its own. Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n Co-authored-by: Claude --- packages/metadata-core/src/index.ts | 6 + .../src/package-upgrade-plan.test.ts | 180 ++++++++ .../metadata-core/src/package-upgrade-plan.ts | 388 ++++++++++++++++++ 3 files changed, 574 insertions(+) create mode 100644 packages/metadata-core/src/package-upgrade-plan.test.ts create mode 100644 packages/metadata-core/src/package-upgrade-plan.ts diff --git a/packages/metadata-core/src/index.ts b/packages/metadata-core/src/index.ts index 4e7c2a0b0cf..112ca09f20d 100644 --- a/packages/metadata-core/src/index.ts +++ b/packages/metadata-core/src/index.ts @@ -156,3 +156,9 @@ export * from './extension-field-patch-targets.js'; // authoring gate the draft verdict and the publish doors run, so a flow the // engine refuses to register no longer reads valid and publishes. export * from './flow-cycle.js'; + +// [epic cloud#3652 Q3] The package upgrade plan — one comparison of an installed +// package version with the incoming one, read by `os diff`, the marketplace +// install door's dry run and the `manifest` service's `planUpgrade`, so the +// preview an installer reads and the comparison a door decides on are one. +export * from './package-upgrade-plan.js'; diff --git a/packages/metadata-core/src/package-upgrade-plan.test.ts b/packages/metadata-core/src/package-upgrade-plan.test.ts new file mode 100644 index 00000000000..384b4ed0240 --- /dev/null +++ b/packages/metadata-core/src/package-upgrade-plan.test.ts @@ -0,0 +1,180 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The package upgrade plan's comparison (epic cloud#3652 Q3): what it lists + * for each shape of change between an installed version and the incoming one. + */ + +import { describe, expect, it } from 'vitest'; +import { comparePackageVersions, isEmptyPackageUpgradePlan, typeChangeCategory } from './package-upgrade-plan.js'; + +type Fields = Record>; + +const side = (fields: Fields, extra: Record = {}) => ({ + objects: [{ name: 'acme_deal', label: 'Deal', fields }], + ...extra, +}); + +describe('fields and objects', () => { + it('an identical pair lists nothing', () => { + const body = side({ title: { type: 'text' } }, { data: [{ object: 'acme_deal', records: [{ name: 'a' }] }] }); + const plan = comparePackageVersions(body, structuredClone(body)); + expect(isEmptyPackageUpgradePlan(plan)).toBe(true); + }); + + it('objects added and removed are listed by name', () => { + const plan = comparePackageVersions( + { objects: [{ name: 'acme_old', fields: {} }, { name: 'acme_kept', fields: {} }] }, + { objects: [{ name: 'acme_kept', fields: {} }, { name: 'acme_new', fields: {} }] }, + ); + expect(plan.objects).toEqual({ added: ['acme_new'], removed: ['acme_old'] }); + }); + + it('a field added carries whether it is required; a field removed whether it is stored', () => { + const plan = comparePackageVersions( + side({ title: { type: 'text' }, total: { type: 'formula' } }), + side({ title: { type: 'text' }, code: { type: 'text', required: true } }), + ); + expect(plan.fields.added).toEqual([{ object: 'acme_deal', field: 'code', type: 'text', required: true, stored: true }]); + expect(plan.fields.removed).toEqual([{ object: 'acme_deal', field: 'total', type: 'formula', required: false, stored: false }]); + // A computed field held no column: its removal is no suspected rename. + expect(plan.suspectedRenames).toEqual([]); + }); +}); + +describe('renames', () => { + it('a field removed and one added of the same type in one object, no renamedFrom → one suspected rename', () => { + const plan = comparePackageVersions(side({ company: { type: 'text' } }), side({ account_name: { type: 'text' } })); + expect(plan.suspectedRenames).toEqual([ + { object: 'acme_deal', field: 'company', type: 'text', candidates: [{ field: 'account_name', type: 'text' }] }, + ]); + expect(plan.declaredRenames).toEqual([]); + // Nothing declares a rename, so both keys are still listed as removed and added. + expect(plan.fields.removed.map((f) => f.field)).toEqual(['company']); + expect(plan.fields.added.map((f) => f.field)).toEqual(['account_name']); + }); + + it("the same pair with renamedFrom: ['company'] on the new field → one declared rename and no suspected rename", () => { + const plan = comparePackageVersions( + side({ company: { type: 'text' } }), + side({ account_name: { type: 'text', renamedFrom: ['company'] } }), + ); + expect(plan.declaredRenames).toEqual([ + { object: 'acme_deal', from: 'company', to: 'account_name', fromType: 'text', toType: 'text' }, + ]); + expect(plan.suspectedRenames).toEqual([]); + expect(plan.fields.added).toEqual([]); + expect(plan.fields.removed).toEqual([]); + expect(plan.typeChanges).toEqual([]); + }); + + it('a declared rename whose type changes is ALSO a type change, judged on its own', () => { + const plan = comparePackageVersions( + side({ amount: { type: 'text' } }), + side({ amount_value: { type: 'number', renamedFrom: ['amount'] } }), + ); + expect(plan.declaredRenames).toEqual([ + { object: 'acme_deal', from: 'amount', to: 'amount_value', fromType: 'text', toType: 'number' }, + ]); + // `needs_confirm`, not `destructive`: the declared rename does not decide + // whether the type change is reported. + expect(plan.typeChanges).toEqual([ + { object: 'acme_deal', field: 'amount_value', from: 'text', to: 'number', category: 'needs_confirm', renamedFrom: 'amount' }, + ]); + }); + + it('a renamedFrom naming a key the installed version does not have declares nothing', () => { + const plan = comparePackageVersions( + side({ company: { type: 'text' } }), + side({ account_name: { type: 'text', renamedFrom: ['firm'] } }), + ); + expect(plan.declaredRenames).toEqual([]); + expect(plan.suspectedRenames).toHaveLength(1); + }); + + it('a renamedFrom naming a key the new version still declares is no rename', () => { + const plan = comparePackageVersions( + side({ company: { type: 'text' } }), + side({ company: { type: 'text' }, account_name: { type: 'text', renamedFrom: ['company'] } }), + ); + expect(plan.declaredRenames).toEqual([]); + expect(plan.fields.added.map((f) => f.field)).toEqual(['account_name']); + }); + + it('one removed field and two added name both as candidates; a linked pair is not a candidate', () => { + const plan = comparePackageVersions( + side({ company: { type: 'text' }, region: { type: 'select' } }), + side({ account_name: { type: 'text' }, area: { type: 'select', renamedFrom: ['region'] }, note: { type: 'textarea' } }), + ); + expect(plan.declaredRenames.map((r) => `${r.from}→${r.to}`)).toEqual(['region→area']); + expect(plan.suspectedRenames).toEqual([ + { + object: 'acme_deal', field: 'company', type: 'text', + candidates: [{ field: 'account_name', type: 'text' }, { field: 'note', type: 'textarea' }], + }, + ]); + }); +}); + +describe('type changes', () => { + it('a type change carries its drift category', () => { + const plan = comparePackageVersions( + side({ stage: { type: 'text' }, score: { type: 'formula' }, total: { type: 'number' } }), + side({ stage: { type: 'select' }, score: { type: 'number' }, total: { type: 'formula' } }), + ); + expect(plan.typeChanges).toEqual([ + { object: 'acme_deal', field: 'stage', from: 'text', to: 'select', category: 'needs_confirm' }, + { object: 'acme_deal', field: 'score', from: 'formula', to: 'number', category: 'safe' }, + { object: 'acme_deal', field: 'total', from: 'number', to: 'formula', category: 'destructive' }, + ]); + }); + + it('the category reads the storage fact, one rule per case', () => { + expect(typeChangeCategory({ type: 'formula' }, { type: 'text' })).toBe('safe'); + expect(typeChangeCategory({ type: 'text' }, { type: 'formula' })).toBe('destructive'); + expect(typeChangeCategory({ type: 'text' }, { type: 'number' })).toBe('needs_confirm'); + }); +}); + +describe('seed datasets', () => { + const records = (n: number) => Array.from({ length: n }, (_, i) => ({ name: `r${i}`, label: `Row ${i}` })); + + it('an upsert dataset change and an ignore dataset change are both listed with their modes', () => { + const plan = comparePackageVersions( + side({}, { data: [ + { object: 'acme_deal', mode: 'upsert', records: records(2) }, + { object: 'acme_stage', mode: 'ignore', records: records(1) }, + ] }), + side({}, { data: [ + { object: 'acme_deal', mode: 'upsert', records: records(3) }, + { object: 'acme_stage', mode: 'ignore', records: [{ name: 'r0', label: 'Renamed' }] }, + ] }), + ); + expect(plan.seeds.changed).toEqual([ + { key: 'acme_deal', object: 'acme_deal', externalId: 'name', mode: 'upsert', records: 3, fromMode: 'upsert', fromRecords: 2, changed: ['records'] }, + { key: 'acme_stage', object: 'acme_stage', externalId: 'name', mode: 'ignore', records: 1, fromMode: 'ignore', fromRecords: 1, changed: ['records'] }, + ]); + }); + + it('a dataset that declares no mode is listed under the spec default', () => { + const plan = comparePackageVersions(side({}), side({}, { data: [{ object: 'acme_deal', records: records(1) }] })); + expect(plan.seeds.added).toEqual([{ key: 'acme_deal', object: 'acme_deal', externalId: 'name', mode: 'ignore', records: 1 }]); + }); + + it('a removed dataset and a mode change are listed', () => { + const plan = comparePackageVersions( + side({}, { data: [{ object: 'acme_deal', mode: 'ignore', records: records(1) }, { object: 'acme_old', records: records(1) }] }), + side({}, { data: [{ object: 'acme_deal', mode: 'upsert', records: records(1) }] }), + ); + expect(plan.seeds.removed.map((d) => d.key)).toEqual(['acme_old']); + expect(plan.seeds.changed).toEqual([expect.objectContaining({ key: 'acme_deal', mode: 'upsert', fromMode: 'ignore', changed: ['mode'] })]); + }); + + it('record key order is not a change', () => { + const plan = comparePackageVersions( + side({}, { data: [{ object: 'acme_deal', records: [{ name: 'a', label: 'A' }] }] }), + side({}, { data: [{ object: 'acme_deal', records: [{ label: 'A', name: 'a' }] }] }), + ); + expect(plan.seeds.changed).toEqual([]); + }); +}); diff --git a/packages/metadata-core/src/package-upgrade-plan.ts b/packages/metadata-core/src/package-upgrade-plan.ts new file mode 100644 index 00000000000..aadfbb2b01e --- /dev/null +++ b/packages/metadata-core/src/package-upgrade-plan.ts @@ -0,0 +1,388 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The package upgrade plan: ONE comparison of the version of a package that is + * installed with the version about to replace it (epic cloud#3652, Phase 2 + * "change preview before install"; ruling Q3 「默认拒绝，需显式映射（推荐）」 and + * D3 「写进清单 renamedFrom」). + * + * ## Why one function + * + * Every reader of "what will this upgrade change" calls this: `os diff` + * (two config files), the install-local door's dry run and its upgrade + * verdict, and the `manifest` service's `planUpgrade`, which a host such as a + * hosted install door calls. Two comparisons would disagree about the same + * pair of versions, and the one an installer saw would not be the one that + * decided. + * + * It is pure: two package bodies in, findings out. It reads no registry, no + * database and no ledger. What only the running environment knows (rows in a + * column, rows a customer edited) is the `manifest` service's half. + * + * ## What it reads + * + * Each side is ONE flat package body, as far as this function is concerned: + * its `objects` (each an object with a `name` and a `fields` map) and its + * `data` (seed datasets). A multi-package artifact is folded by the caller + * with the platform's one collection reader (`resolveStackCollection` in + * `@objectstack/core`; `os diff` folds through the CLI's union stack), so this + * module holds no second reading of `packages[]`. + * + * ## What it reports + * + * - objects added and removed, by name; + * - fields added and removed, by key, within an object both versions declare; + * - type changes: a field both versions declare whose `type` differs, with a + * drift category ({@link typeChangeCategory}); + * - declared renames: a field of the new version whose `renamedFrom` names a + * field of the installed version that the new version no longer declares. + * The pair is a rename, so neither key is listed as added or removed. When + * the two types differ, the rename is ALSO a type change, judged on its own: + * a declared rename never stands in for a type-change confirmation + * (repo:cloud#3's scope note on the card); + * - suspected renames: within one object, a stored field removed while a + * stored field is added, with no `renamedFrom` linking them. One finding per + * removed field, naming every unlinked added field as a candidate. Both keys + * stay listed as removed and added: nothing declares otherwise; + * - seed datasets added, removed and changed, each with its mode. + * + * ⛔ It decides nothing. Which findings stop an install is the door's call. + */ + +import { SeedSchema, isVirtualSearchField } from '@objectstack/spec/data'; +import { canonicalize } from './canonicalize.js'; + +/** + * The drift categories of `@objectstack/driver-sql`'s `schema-drift.ts` + * (`DriftCategory`), the same three words: this package cannot depend on a + * driver, so the union is restated here and held equal to the driver's by a + * type-level pin in the CLI's tests, which depend on both. + * + * - `safe` loosening that cannot lose data and cannot fail; + * - `needs_confirm` a change a human should look at, that destroys nothing; + * - `destructive` a drop or tightening that can lose data or fail. + */ +export type UpgradeDriftCategory = 'safe' | 'needs_confirm' | 'destructive'; + +/** One side of the comparison: a flat package body's two collections. */ +export interface PackageUpgradeSide { + /** The body's object definitions (`{ name, fields }`). */ + objects?: readonly unknown[]; + /** The body's seed datasets (`SeedSchema`). */ + data?: readonly unknown[]; +} + +/** A field one side declares and the other does not. */ +export interface UpgradePlanField { + object: string; + field: string; + type?: string; + /** `required: true` on the declaration. */ + required: boolean; + /** The field has a stored column (not a computed-on-read type). */ + stored: boolean; +} + +/** A field whose declared type differs between the two versions. */ +export interface UpgradePlanTypeChange { + object: string; + /** The field's key in the NEW version. */ + field: string; + from: string; + to: string; + category: UpgradeDriftCategory; + /** Set when the change rides on a declared rename: the installed key. */ + renamedFrom?: string; +} + +/** A rename the new version declares with `renamedFrom`. */ +export interface UpgradePlanDeclaredRename { + object: string; + /** The installed key. */ + from: string; + /** The new key. */ + to: string; + fromType?: string; + toType?: string; +} + +/** A stored field removed while stored fields are added, with nothing linking them. */ +export interface UpgradePlanSuspectedRename { + object: string; + /** The removed (installed) key. */ + field: string; + type?: string; + /** Every added stored field of the same object no `renamedFrom` links. */ + candidates: Array<{ field: string; type?: string }>; +} + +/** A seed dataset, as the plan names it. */ +export interface UpgradePlanSeedDataset { + /** `object`, plus the match key and the locale scope when they are not the defaults. */ + key: string; + object: string; + externalId: string | string[]; + /** The mode, the spec default (`ignore`) when the dataset declares none. */ + mode: string; + records: number; +} + +/** A dataset both versions declare under one key, with what differs. */ +export interface UpgradePlanSeedChange extends UpgradePlanSeedDataset { + /** The installed version's mode. */ + fromMode: string; + /** The installed version's record count. */ + fromRecords: number; + /** What differs: the records, the mode, the column policies, the environments. */ + changed: Array<'records' | 'mode' | 'columns' | 'env'>; +} + +export interface PackageUpgradePlan { + objects: { added: string[]; removed: string[] }; + fields: { added: UpgradePlanField[]; removed: UpgradePlanField[] }; + typeChanges: UpgradePlanTypeChange[]; + declaredRenames: UpgradePlanDeclaredRename[]; + suspectedRenames: UpgradePlanSuspectedRename[]; + seeds: { added: UpgradePlanSeedDataset[]; removed: UpgradePlanSeedDataset[]; changed: UpgradePlanSeedChange[] }; +} + +type Bag = Record; + +function isBag(value: unknown): value is Bag { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Objects by name, the LAST declaration of a name winning (`os diff`'s reading). */ +function objectsByName(side: PackageUpgradeSide | undefined): Map { + const out = new Map(); + for (const item of side?.objects ?? []) { + if (isBag(item) && typeof item.name === 'string' && item.name !== '') out.set(item.name, item); + } + return out; +} + +function fieldsOf(object: Bag | undefined): Map { + const out = new Map(); + const fields = object?.fields; + if (!isBag(fields)) return out; + for (const [key, def] of Object.entries(fields)) out.set(key, isBag(def) ? def : {}); + return out; +} + +function typeOf(field: Bag | undefined): string | undefined { + return typeof field?.type === 'string' && field.type !== '' ? field.type : undefined; +} + +/** Has the field a stored column? The spec's storage predicate, never a type list kept here. */ +function isStored(field: Bag | undefined): boolean { + return !isVirtualSearchField({ type: typeOf(field) } as Parameters[0]); +} + +function describeField(object: string, field: string, def: Bag | undefined): UpgradePlanField { + const type = typeOf(def); + return { object, field, ...(type !== undefined ? { type } : {}), required: def?.required === true, stored: isStored(def) }; +} + +/** + * The drift category of a declared type change, in `schema-drift.ts`'s + * vocabulary and by its reading of the same physical facts. A package upgrade + * syncs the schema ADDITIVELY: it adds missing columns and never alters or + * drops one, so a retyped field keeps its existing column. + * + * - The installed field had no stored column (computed on read): there is + * nothing stored to lose, and the new column is one the additive sync adds. + * `safe`. + * - The new field has no stored column and the installed one had: the column + * stops being declared, which the drift detector reports as an orphan whose + * operation is a column drop. `destructive`. + * - Both stored: the column keeps its old type under a declaration of another + * type. The detector reports a base-type divergence as report-only, applied + * by no reconciler (`manual_column_type_change`). `needs_confirm`. + */ +export function typeChangeCategory(from: { type?: unknown }, to: { type?: unknown }): UpgradeDriftCategory { + const fromStored = isStored(from as Bag); + const toStored = isStored(to as Bag); + if (!fromStored) return 'safe'; + if (!toStored) return 'destructive'; + return 'needs_confirm'; +} + +// ── seeds ─────────────────────────────────────────────────────────────── + +const DEFAULT_SEED_MODE: string = SeedSchema.shape.mode.parse(undefined); +const DEFAULT_SEED_EXTERNAL_ID: string | string[] = SeedSchema.shape.externalId.parse(undefined); + +interface ReadDataset extends UpgradePlanSeedDataset { + raw: Bag; +} + +function stableText(value: unknown): string { + try { + return canonicalize(value); + } catch { + // A value canonical JSON refuses (NaN, Infinity) still compares by its + // plain serialisation rather than throwing the whole plan away. + return JSON.stringify(value) ?? 'undefined'; + } +} + +function datasetsOf(side: PackageUpgradeSide | undefined): Map { + const out = new Map(); + for (const item of side?.data ?? []) { + if (!isBag(item) || typeof item.object !== 'string' || item.object === '') continue; + const externalId = typeof item.externalId === 'string' || Array.isArray(item.externalId) + ? (item.externalId as string | string[]) + : DEFAULT_SEED_EXTERNAL_ID; + const mode = typeof item.mode === 'string' ? item.mode : DEFAULT_SEED_MODE; + const locale = Array.isArray(item.locale) ? [...item.locale].map(String).sort().join(',') : ''; + let key = item.object; + if (stableText(externalId) !== stableText(DEFAULT_SEED_EXTERNAL_ID)) key += ` by ${Array.isArray(externalId) ? externalId.join('+') : externalId}`; + if (locale !== '') key += ` [${locale}]`; + // Two datasets under one key in one version are told apart by position. + let unique = key; + for (let n = 2; out.has(unique); n++) unique = `${key} #${n}`; + const records = Array.isArray(item.records) ? item.records.length : 0; + out.set(unique, { key: unique, object: item.object, externalId, mode, records, raw: item }); + } + return out; +} + +function publicDataset(d: ReadDataset): UpgradePlanSeedDataset { + return { key: d.key, object: d.object, externalId: d.externalId, mode: d.mode, records: d.records }; +} + +function compareSeeds(installed: PackageUpgradeSide | undefined, incoming: PackageUpgradeSide | undefined): PackageUpgradePlan['seeds'] { + const before = datasetsOf(installed); + const after = datasetsOf(incoming); + const seeds: PackageUpgradePlan['seeds'] = { added: [], removed: [], changed: [] }; + for (const [key, d] of before) if (!after.has(key)) seeds.removed.push(publicDataset(d)); + for (const [key, d] of after) { + const prior = before.get(key); + if (!prior) { + seeds.added.push(publicDataset(d)); + continue; + } + const changed: UpgradePlanSeedChange['changed'] = []; + if (stableText(prior.raw.records ?? []) !== stableText(d.raw.records ?? [])) changed.push('records'); + if (prior.mode !== d.mode) changed.push('mode'); + if (stableText(prior.raw.columns ?? {}) !== stableText(d.raw.columns ?? {})) changed.push('columns'); + if (stableText(prior.raw.env ?? null) !== stableText(d.raw.env ?? null)) changed.push('env'); + if (changed.length > 0) { + seeds.changed.push({ ...publicDataset(d), fromMode: prior.mode, fromRecords: prior.records, changed }); + } + } + return seeds; +} + +// ── the comparison ────────────────────────────────────────────────────── + +/** + * Compare the installed version of a package with the incoming one. Lists + * are ordered as the versions declare: removed objects, fields and type + * changes in the installed version's order, added ones in the incoming + * version's order, objects one after another. + */ +export function comparePackageVersions(installed: PackageUpgradeSide | undefined, incoming: PackageUpgradeSide | undefined): PackageUpgradePlan { + const before = objectsByName(installed); + const after = objectsByName(incoming); + const plan: PackageUpgradePlan = { + objects: { added: [], removed: [] }, + fields: { added: [], removed: [] }, + typeChanges: [], + declaredRenames: [], + suspectedRenames: [], + seeds: compareSeeds(installed, incoming), + }; + + for (const name of before.keys()) if (!after.has(name)) plan.objects.removed.push(name); + for (const name of after.keys()) if (!before.has(name)) plan.objects.added.push(name); + + for (const [object, beforeObject] of before) { + const afterObject = after.get(object); + if (!afterObject) continue; + const beforeFields = fieldsOf(beforeObject); + const afterFields = fieldsOf(afterObject); + + // Declared renames: a NEW key whose `renamedFrom` names an installed key + // the new version no longer declares. The first such entry is the rename + // (the spec refuses an old key two fields claim, and a field renamed + // twice names at most one key this installed version has). + const renamedTo = new Map(); // installed key → new key + for (const [key, def] of afterFields) { + if (beforeFields.has(key) || !Array.isArray(def.renamedFrom)) continue; + for (const old of def.renamedFrom) { + if (typeof old !== 'string' || !beforeFields.has(old) || afterFields.has(old) || renamedTo.has(old)) continue; + renamedTo.set(old, key); + break; + } + } + const renameTargets = new Set(renamedTo.values()); + + const removed: string[] = []; + for (const key of beforeFields.keys()) { + if (afterFields.has(key) || renamedTo.has(key)) continue; + removed.push(key); + plan.fields.removed.push(describeField(object, key, beforeFields.get(key))); + } + const added: string[] = []; + for (const key of afterFields.keys()) { + if (beforeFields.has(key) || renameTargets.has(key)) continue; + added.push(key); + plan.fields.added.push(describeField(object, key, afterFields.get(key))); + } + + for (const [key, beforeDef] of beforeFields) { + const target = afterFields.has(key) ? key : renamedTo.get(key); + if (target === undefined) continue; + const afterDef = afterFields.get(target); + const from = typeOf(beforeDef); + const to = typeOf(afterDef); + if (target !== key) { + plan.declaredRenames.push({ + object, from: key, to: target, + ...(from !== undefined ? { fromType: from } : {}), + ...(to !== undefined ? { toType: to } : {}), + }); + } + if (from !== undefined && to !== undefined && from !== to) { + plan.typeChanges.push({ + object, field: target, from, to, + category: typeChangeCategory(beforeDef, afterDef ?? {}), + ...(target !== key ? { renamedFrom: key } : {}), + }); + } + } + + const storedAdded = added.filter((key) => isStored(afterFields.get(key))); + if (storedAdded.length > 0) { + for (const key of removed) { + const def = beforeFields.get(key); + if (!isStored(def)) continue; + const type = typeOf(def); + plan.suspectedRenames.push({ + object, field: key, ...(type !== undefined ? { type } : {}), + candidates: storedAdded.map((candidate) => { + const candidateType = typeOf(afterFields.get(candidate)); + return { field: candidate, ...(candidateType !== undefined ? { type: candidateType } : {}) }; + }), + }); + } + } + } + + return plan; +} + +/** Does the plan list nothing at all? */ +export function isEmptyPackageUpgradePlan(plan: PackageUpgradePlan): boolean { + return plan.objects.added.length === 0 + && plan.objects.removed.length === 0 + && plan.fields.added.length === 0 + && plan.fields.removed.length === 0 + && plan.typeChanges.length === 0 + && plan.declaredRenames.length === 0 + && plan.suspectedRenames.length === 0 + && plan.seeds.added.length === 0 + && plan.seeds.removed.length === 0 + && plan.seeds.changed.length === 0; +} From 60c95956d0256024a7e3070cf72e1613e5174ac1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:03:16 +0000 Subject: [PATCH 2/7] test(cli): pin os diff's current output before it moves onto the upgrade plan `diffStacks` lifts the command's comparison out of `run()` unchanged, and the new test pins its output byte for byte (entry order and each entry's key order) over every shape it reports today, plus a type-level pin that the upgrade plan's three drift words equal driver-sql's `DriftCategory`. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- packages/cli/src/commands/diff.ts | 58 +++++++------ packages/cli/test/diff-upgrade-plan.test.ts | 93 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 24 deletions(-) create mode 100644 packages/cli/test/diff-upgrade-plan.test.ts diff --git a/packages/cli/src/commands/diff.ts b/packages/cli/src/commands/diff.ts index 6e0cc3a8d56..998d7115c6b 100644 --- a/packages/cli/src/commands/diff.ts +++ b/packages/cli/src/commands/diff.ts @@ -20,7 +20,7 @@ import { // ─── Types ────────────────────────────────────────────────────────── -interface DiffEntry { +export interface DiffEntry { type: 'added' | 'removed' | 'modified'; category: string; name: string; @@ -156,6 +156,38 @@ function diffNamedArrays( return entries; } +/** + * Every change between two folded stacks, in the order the command prints + * them: objects with their field-level changes first, then each name-level + * category. + */ +export function diffStacks(beforeConfig: any, afterConfig: any): DiffEntry[] { + const allDiffs: DiffEntry[] = []; + + // Objects (with field-level diff) + allDiffs.push(...diffNamedArrays(beforeConfig.objects, afterConfig.objects, 'objects', true)); + + // Views, Flows, Agents, Apps (name-level diff) + const simpleCats: Array<{ key: string; label: string }> = [ + { key: 'views', label: 'views' }, + { key: 'flows', label: 'flows' }, + { key: 'agents', label: 'agents' }, + { key: 'apps', label: 'apps' }, + { key: 'dashboards', label: 'dashboards' }, + { key: 'actions', label: 'actions' }, + { key: 'workflows', label: 'workflows' }, + { key: 'apis', label: 'apis' }, + { key: 'positions', label: 'positions' }, + ]; + + for (const cat of simpleCats) { + allDiffs.push( + ...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label, false), + ); + } + return allDiffs; +} + // ─── Command ──────────────────────────────────────────────────────── export default class Diff extends Command { @@ -229,29 +261,7 @@ export default class Diff extends Command { } // ── Diff all categories ── - const allDiffs: DiffEntry[] = []; - - // Objects (with field-level diff) - allDiffs.push(...diffNamedArrays(beforeConfig.objects, afterConfig.objects, 'objects', true)); - - // Views, Flows, Agents, Apps (name-level diff) - const simpleCats: Array<{ key: string; label: string }> = [ - { key: 'views', label: 'views' }, - { key: 'flows', label: 'flows' }, - { key: 'agents', label: 'agents' }, - { key: 'apps', label: 'apps' }, - { key: 'dashboards', label: 'dashboards' }, - { key: 'actions', label: 'actions' }, - { key: 'workflows', label: 'workflows' }, - { key: 'apis', label: 'apis' }, - { key: 'positions', label: 'positions' }, - ]; - - for (const cat of simpleCats) { - allDiffs.push( - ...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label, false), - ); - } + const allDiffs = diffStacks(beforeConfig, afterConfig); // ── Filter ── const diffs = flags['breaking-only'] diff --git a/packages/cli/test/diff-upgrade-plan.test.ts b/packages/cli/test/diff-upgrade-plan.test.ts new file mode 100644 index 00000000000..bdb0b70e166 --- /dev/null +++ b/packages/cli/test/diff-upgrade-plan.test.ts @@ -0,0 +1,93 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os diff` reads its object and field changes from the package upgrade plan + * (`comparePackageVersions`, `@objectstack/metadata-core`), the comparison the + * install door and the `manifest` service's `planUpgrade` read too (epic + * cloud#3652 Q3). + * + * The first block pins today's output, byte for byte (`JSON.stringify`, so the + * key order of each entry is pinned as well — `--json` prints these objects), + * over every shape the command reported before it moved onto the plan. It was + * written against the command's own comparison and stayed green across the + * move. The second block pins what the plan adds: renames, suspected renames + * and seed datasets. + */ + +import { describe, expect, it } from 'vitest'; +import type { DriftCategory } from '@objectstack/driver-sql'; +import type { UpgradeDriftCategory } from '@objectstack/metadata-core'; +import { diffStacks } from '../src/commands/diff.js'; + +// The plan restates `schema-drift.ts`'s three drift words, since +// `@objectstack/metadata-core` cannot depend on a driver; this package depends +// on both, so the two unions are held equal here. A word added to or dropped +// from either side fails `tsc` on this line. +type Equal = (() => T extends A ? 1 : 2) extends (() => T extends B ? 1 : 2) ? true : false; +const driftWordsAgree: Equal = true; + +const BEFORE = { + objects: [ + { name: 'acme_deal', label: 'Deal', fields: { title: { type: 'text' }, amount: { type: 'number' }, stage: { type: 'text' } } }, + { name: 'acme_lead', label: 'Lead', fields: { legacy: { type: 'text' }, name: { type: 'text' } } }, + { name: 'acme_old', label: 'Old', fields: { x: { type: 'text' } } }, + { name: 'acme_dup', fields: { a: { type: 'text' } } }, + { name: 'acme_dup', fields: { b: { type: 'text' } } }, + ], + views: [{ name: 'deal_list' }, { name: 'old_view' }], + flows: [{ name: 'f1' }], + positions: [{ name: 'p1' }], +}; + +const AFTER = { + objects: [ + { + name: 'acme_deal', + label: 'Deals', + fields: { + title: { type: 'text' }, + amount: { type: 'currency' }, + stage: { type: 'text' }, + code: { type: 'text', required: true }, + note: { type: 'textarea' }, + }, + }, + { name: 'acme_lead', label: 'Lead', fields: { name: { type: 'text' } } }, + { name: 'acme_new', fields: {} }, + { name: 'acme_dup', fields: { b: { type: 'text' } } }, + ], + views: [{ name: 'deal_list' }, { name: 'new_view' }], + flows: [], + apps: [{ name: 'a1' }], + positions: [{ name: 'p1' }], +}; + +describe('os diff — the output it printed before the plan, unchanged', () => { + it('every shape, in order, with each entry\'s key order', () => { + expect(driftWordsAgree).toBe(true); + const expected = [ + { type: 'removed', category: 'objects', name: 'acme_old', breaking: true }, + { type: 'added', category: 'objects', name: 'acme_new', breaking: false }, + { type: 'added', category: 'objects.acme_deal.fields', name: 'code', breaking: true, detail: 'required field added' }, + { type: 'added', category: 'objects.acme_deal.fields', name: 'note', breaking: false, detail: 'optional field added' }, + { type: 'modified', category: 'objects.acme_deal.fields', name: 'amount', breaking: true, detail: 'type changed: number → currency' }, + { type: 'modified', category: 'objects', name: 'acme_deal', breaking: false, detail: 'label changed: "Deal" → "Deals"' }, + { type: 'removed', category: 'objects.acme_lead.fields', name: 'legacy', breaking: true, detail: 'field removed' }, + { type: 'removed', category: 'views', name: 'old_view', breaking: true }, + { type: 'added', category: 'views', name: 'new_view', breaking: false }, + { type: 'removed', category: 'flows', name: 'f1', breaking: true }, + { type: 'added', category: 'apps', name: 'a1', breaking: false }, + ]; + expect(JSON.stringify(diffStacks(BEFORE, AFTER))).toBe(JSON.stringify(expected)); + }); + + it('an identical pair reports nothing', () => { + expect(diffStacks(BEFORE, structuredClone(BEFORE))).toEqual([]); + }); + + it('a stack with no objects on either side reports only its name-level categories', () => { + expect(diffStacks({ views: [{ name: 'v' }] }, {})).toEqual([ + { type: 'removed', category: 'views', name: 'v', breaking: true }, + ]); + }); +}); From 3714e769bd538db52b6bf64834fb8e19f3de212c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:07:54 +0000 Subject: [PATCH 3/7] feat(cli): os diff reads objects, fields and seeds from the package upgrade plan `os diff` now builds its object and field entries from `comparePackageVersions` (`@objectstack/metadata-core`), the comparison the install door's dry run and the `manifest` service read, so the CLI and an install cannot disagree about one pair of versions. The output for every shape it reported before is unchanged (pinned before the move). New with the plan: a declared `renamedFrom` rename, a suspected rename on the removed key, and seed datasets added, removed and changed, each with its mode. `seedModeOverwritesMatchedRows` names the two modes whose replay writes over a matched row, for the CLI's wording and the environment half's count. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- packages/cli/src/commands/diff.ts | 181 ++++++++++-------- packages/cli/test/diff-upgrade-plan.test.ts | 63 ++++++ .../metadata-core/src/package-upgrade-plan.ts | 14 +- 3 files changed, 174 insertions(+), 84 deletions(-) diff --git a/packages/cli/src/commands/diff.ts b/packages/cli/src/commands/diff.ts index 998d7115c6b..bbe219fef65 100644 --- a/packages/cli/src/commands/diff.ts +++ b/packages/cli/src/commands/diff.ts @@ -2,6 +2,7 @@ import { Args, Command, Flags } from '@oclif/core'; import chalk from 'chalk'; +import { comparePackageVersions, seedModeOverwritesMatchedRows } from '@objectstack/metadata-core'; import { loadConfig } from '../utils/config.js'; import { authoringRuleUnionStack } from '../utils/stack-collections.js'; import { @@ -39,24 +40,10 @@ function getNames(items: any[] | undefined): Map { return map; } -function getFieldNames(obj: any): string[] { - if (!obj?.fields || typeof obj.fields !== 'object') return []; - return Object.keys(obj.fields); -} - -function getFieldType(obj: any, fieldName: string): string | undefined { - return obj?.fields?.[fieldName]?.type; -} - -function isFieldRequired(obj: any, fieldName: string): boolean { - return obj?.fields?.[fieldName]?.required === true; -} - function diffNamedArrays( beforeItems: any[] | undefined, afterItems: any[] | undefined, category: string, - detectFieldChanges: boolean, ): DiffEntry[] { const entries: DiffEntry[] = []; const beforeMap = getNames(beforeItems); @@ -86,86 +73,115 @@ function diffNamedArrays( } } - // Modified items — field-level diff for objects - if (detectFieldChanges) { - for (const [name, beforeObj] of beforeMap) { - const afterObj = afterMap.get(name); - if (!afterObj) continue; - - const beforeFields = getFieldNames(beforeObj); - const afterFields = getFieldNames(afterObj); - const beforeSet = new Set(beforeFields); - const afterSet = new Set(afterFields); - - // Removed fields - for (const f of beforeFields) { - if (!afterSet.has(f)) { - entries.push({ - type: 'removed', - category: `${category}.${name}.fields`, - name: f, - breaking: true, - detail: 'field removed', - }); - } - } + return entries; +} - // Added fields - for (const f of afterFields) { - if (!beforeSet.has(f)) { - const breaking = isFieldRequired(afterObj, f); - entries.push({ - type: 'added', - category: `${category}.${name}.fields`, - name: f, - breaking, - detail: breaking ? 'required field added' : 'optional field added', - }); - } - } +function seedModeNote(mode: string): string { + if (seedModeOverwritesMatchedRows(mode)) return '; a replay writes its columns over the rows it matches'; + if (mode === 'replace') return '; a replay deletes every row of the object and inserts these'; + return ''; +} - // Type changes - for (const f of beforeFields) { - if (!afterSet.has(f)) continue; - const oldType = getFieldType(beforeObj, f); - const newType = getFieldType(afterObj, f); - if (oldType && newType && oldType !== newType) { - entries.push({ - type: 'modified', - category: `${category}.${name}.fields`, - name: f, - breaking: true, - detail: `type changed: ${oldType} → ${newType}`, - }); - } - } +/** + * Objects, fields and seed datasets, read from the package upgrade plan + * (`comparePackageVersions`, `@objectstack/metadata-core`) — the comparison + * the install door's dry run and the `manifest` service's `planUpgrade` read, + * so this command and an install cannot disagree about one pair of versions. + * + * Printed per object as before: removed fields, added fields, then (new with + * the plan) declared renames, type changes, then suspected renames, then the + * object's label change. A suspected rename's two keys stay listed as removed + * and added, as the plan lists them: nothing declares otherwise. + */ +function diffObjectsAndSeeds(beforeConfig: any, afterConfig: any): DiffEntry[] { + const plan = comparePackageVersions(beforeConfig, afterConfig); + const entries: DiffEntry[] = []; - // Label / ownership changes on the object itself - if (beforeObj.label !== afterObj.label) { - entries.push({ - type: 'modified', - category, - name, - breaking: false, - detail: `label changed: "${beforeObj.label ?? '(none)'}" → "${afterObj.label ?? '(none)'}"`, - }); - } + for (const name of plan.objects.removed) entries.push({ type: 'removed', category: 'objects', name, breaking: true }); + for (const name of plan.objects.added) entries.push({ type: 'added', category: 'objects', name, breaking: false }); + + const afterObjects = getNames(afterConfig.objects); + for (const [name, beforeObj] of getNames(beforeConfig.objects)) { + const afterObj = afterObjects.get(name); + if (!afterObj) continue; + const category = `objects.${name}.fields`; + const mine = (list: T[]): T[] => list.filter((item) => item.object === name); + + for (const f of mine(plan.fields.removed)) { + entries.push({ type: 'removed', category, name: f.field, breaking: true, detail: 'field removed' }); + } + for (const f of mine(plan.fields.added)) { + entries.push({ + type: 'added', + category, + name: f.field, + breaking: f.required, + detail: f.required ? 'required field added' : 'optional field added', + }); + } + for (const r of mine(plan.declaredRenames)) { + entries.push({ type: 'modified', category, name: r.to, breaking: true, detail: `renamed from ${r.from}, declared by renamedFrom` }); + } + for (const t of mine(plan.typeChanges)) { + entries.push({ type: 'modified', category, name: t.field, breaking: true, detail: `type changed: ${t.from} → ${t.to}` }); + } + for (const s of mine(plan.suspectedRenames)) { + entries.push({ + type: 'modified', + category, + name: s.field, + breaking: true, + detail: `suspected rename to ${s.candidates.map((c) => c.field).join(' or ')}: no renamedFrom declares it, ` + + 'so the new key installs as a new empty column', + }); + } + + // Label / ownership changes on the object itself + if (beforeObj.label !== afterObj.label) { + entries.push({ + type: 'modified', + category: 'objects', + name, + breaking: false, + detail: `label changed: "${beforeObj.label ?? '(none)'}" → "${afterObj.label ?? '(none)'}"`, + }); } } + for (const d of plan.seeds.removed) { + entries.push({ type: 'removed', category: 'data', name: d.key, breaking: false, detail: `seed dataset removed, mode ${d.mode}` }); + } + for (const d of plan.seeds.added) { + entries.push({ + type: 'added', + category: 'data', + name: d.key, + breaking: false, + detail: `seed dataset added: ${d.records} record(s); mode ${d.mode}${seedModeNote(d.mode)}`, + }); + } + for (const d of plan.seeds.changed) { + const mode = d.fromMode === d.mode ? d.mode : `${d.fromMode} → ${d.mode}`; + entries.push({ + type: 'modified', + category: 'data', + name: d.key, + breaking: false, + detail: `seed dataset changed: ${d.changed.join(', ')}; mode ${mode}${seedModeNote(d.mode)}`, + }); + } + return entries; } /** * Every change between two folded stacks, in the order the command prints * them: objects with their field-level changes first, then each name-level - * category. + * category, then seed datasets. */ export function diffStacks(beforeConfig: any, afterConfig: any): DiffEntry[] { - const allDiffs: DiffEntry[] = []; - - // Objects (with field-level diff) - allDiffs.push(...diffNamedArrays(beforeConfig.objects, afterConfig.objects, 'objects', true)); + const plan = diffObjectsAndSeeds(beforeConfig, afterConfig); + const allDiffs: DiffEntry[] = plan.filter((entry) => entry.category !== 'data'); // Views, Flows, Agents, Apps (name-level diff) const simpleCats: Array<{ key: string; label: string }> = [ @@ -181,10 +197,9 @@ export function diffStacks(beforeConfig: any, afterConfig: any): DiffEntry[] { ]; for (const cat of simpleCats) { - allDiffs.push( - ...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label, false), - ); + allDiffs.push(...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label)); } + allDiffs.push(...plan.filter((entry) => entry.category === 'data')); return allDiffs; } diff --git a/packages/cli/test/diff-upgrade-plan.test.ts b/packages/cli/test/diff-upgrade-plan.test.ts index bdb0b70e166..5a59479d104 100644 --- a/packages/cli/test/diff-upgrade-plan.test.ts +++ b/packages/cli/test/diff-upgrade-plan.test.ts @@ -91,3 +91,66 @@ describe('os diff — the output it printed before the plan, unchanged', () => { ]); }); }); + +describe('os diff — what the upgrade plan adds', () => { + const lead = (fields: Record, data?: unknown[]) => ({ + objects: [{ name: 'acme_lead', label: 'Lead', fields }], + ...(data ? { data } : {}), + }); + + it('a field removed and one added with no renamedFrom: both listed, and a suspected rename on the removed key', () => { + expect(diffStacks(lead({ company: { type: 'text' } }), lead({ account_name: { type: 'text' } }))).toEqual([ + { type: 'removed', category: 'objects.acme_lead.fields', name: 'company', breaking: true, detail: 'field removed' }, + { type: 'added', category: 'objects.acme_lead.fields', name: 'account_name', breaking: false, detail: 'optional field added' }, + { + type: 'modified', category: 'objects.acme_lead.fields', name: 'company', breaking: true, + detail: 'suspected rename to account_name: no renamedFrom declares it, so the new key installs as a new empty column', + }, + ]); + }); + + it('the same pair with renamedFrom: one rename, no removed, added or suspected entry', () => { + expect(diffStacks( + lead({ company: { type: 'text' } }), + lead({ account_name: { type: 'text', renamedFrom: ['company'] } }), + )).toEqual([ + { type: 'modified', category: 'objects.acme_lead.fields', name: 'account_name', breaking: true, detail: 'renamed from company, declared by renamedFrom' }, + ]); + }); + + it('a declared rename that changes type prints the type change too', () => { + expect(diffStacks( + lead({ amount: { type: 'text' } }), + lead({ amount_value: { type: 'number', renamedFrom: ['amount'] } }), + ).map((e) => e.detail)).toEqual(['renamed from amount, declared by renamedFrom', 'type changed: text → number']); + }); + + it('seed datasets, after every other category, each with its mode', () => { + const before = lead({}, [ + { object: 'acme_stage', mode: 'upsert', records: [{ name: 'new' }] }, + { object: 'acme_region', records: [{ name: 'emea' }] }, + { object: 'acme_old', mode: 'insert', records: [{ name: 'x' }] }, + ]); + const after = { + ...lead({}, [ + { object: 'acme_stage', mode: 'upsert', records: [{ name: 'new' }, { name: 'won' }] }, + { object: 'acme_region', records: [{ name: 'apac' }] }, + { object: 'acme_tier', mode: 'replace', records: [{ name: 'gold' }] }, + ]), + views: [{ name: 'lead_list' }], + }; + expect(diffStacks(before, after)).toEqual([ + { type: 'added', category: 'views', name: 'lead_list', breaking: false }, + { type: 'removed', category: 'data', name: 'acme_old', breaking: false, detail: 'seed dataset removed, mode insert' }, + { + type: 'added', category: 'data', name: 'acme_tier', breaking: false, + detail: 'seed dataset added: 1 record(s); mode replace; a replay deletes every row of the object and inserts these', + }, + { + type: 'modified', category: 'data', name: 'acme_stage', breaking: false, + detail: 'seed dataset changed: records; mode upsert; a replay writes its columns over the rows it matches', + }, + { type: 'modified', category: 'data', name: 'acme_region', breaking: false, detail: 'seed dataset changed: records; mode ignore' }, + ]); + }); +}); diff --git a/packages/metadata-core/src/package-upgrade-plan.ts b/packages/metadata-core/src/package-upgrade-plan.ts index aadfbb2b01e..27c0ccee74b 100644 --- a/packages/metadata-core/src/package-upgrade-plan.ts +++ b/packages/metadata-core/src/package-upgrade-plan.ts @@ -39,7 +39,7 @@ * The pair is a rename, so neither key is listed as added or removed. When * the two types differ, the rename is ALSO a type change, judged on its own: * a declared rename never stands in for a type-change confirmation - * (repo:cloud#3's scope note on the card); + * (the card's scope note, objectstack#30884 comment 6685166393); * - suspected renames: within one object, a stored field removed while a * stored field is added, with no `renamedFrom` linking them. One finding per * removed field, naming every unlinked added field as a candidate. Both keys @@ -212,6 +212,18 @@ export function typeChangeCategory(from: { type?: unknown }, to: { type?: unknow const DEFAULT_SEED_MODE: string = SeedSchema.shape.mode.parse(undefined); const DEFAULT_SEED_EXTERNAL_ID: string | string[] = SeedSchema.shape.externalId.parse(undefined); +/** + * Does a replay of a dataset in this mode write its columns over a row it + * MATCHED by `externalId`? `upsert` and `update` do (a column declaring + * `applyOn: 'create'` excepted); `insert` and `ignore` never touch a matched + * row, and `replace` deletes every row before it inserts, so it matches none. + * The rows a customer edited that an upgrade's replay would overwrite are + * counted for these modes only. + */ +export function seedModeOverwritesMatchedRows(mode: string): boolean { + return mode === 'upsert' || mode === 'update'; +} + interface ReadDataset extends UpgradePlanSeedDataset { raw: Bag; } From 091fbe87e99fab3b4c2bd9b3f8aabcdf60502168 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:21:37 +0000 Subject: [PATCH 4/7] =?UTF-8?q?feat(objectql):=20planUpgrade=20=E2=80=94?= =?UTF-8?q?=20the=20upgrade=20plan=20with=20what=20only=20the=20environmen?= =?UTF-8?q?t=20knows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `manifest` service gains `planUpgrade(installed, incoming)`: the package upgrade plan (`comparePackageVersions`, each side folded with core's `resolveStackCollection`) plus two read-only readings, across every organization with the system context: - rows holding a value in each stored column the upgrade removes, renames (declared or suspected) or retypes, and rows of each object it removes; - for each `upsert`/`update` dataset both versions declare, the rows a customer edited that the seed replay overwrites: a column the replay writes whose stored value differs from the installed seed value and from the incoming one, compared as the replay compares (temporal and blank storage forms; credential, file, reference, expression and `applyOn: 'create'` columns never compared). A failed reading answers null, never zero. metadata-core exports the plan's dataset reader (`upgradePlanSeedDatasets`) so both halves pair datasets by one key. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- .../src/package-upgrade-plan.test.ts | 23 +- .../metadata-core/src/package-upgrade-plan.ts | 21 +- .../src/package-upgrade-environment.test.ts | 207 ++++++++++ .../src/package-upgrade-environment.ts | 390 ++++++++++++++++++ packages/objectql/src/plugin.ts | 11 + 5 files changed, 644 insertions(+), 8 deletions(-) create mode 100644 packages/objectql/src/package-upgrade-environment.test.ts create mode 100644 packages/objectql/src/package-upgrade-environment.ts diff --git a/packages/metadata-core/src/package-upgrade-plan.test.ts b/packages/metadata-core/src/package-upgrade-plan.test.ts index 384b4ed0240..0b6cba66c55 100644 --- a/packages/metadata-core/src/package-upgrade-plan.test.ts +++ b/packages/metadata-core/src/package-upgrade-plan.test.ts @@ -6,7 +6,14 @@ */ import { describe, expect, it } from 'vitest'; -import { comparePackageVersions, isEmptyPackageUpgradePlan, typeChangeCategory } from './package-upgrade-plan.js'; +import { SeedMode } from '@objectstack/spec/data'; +import { + comparePackageVersions, + isEmptyPackageUpgradePlan, + seedModeOverwritesMatchedRows, + typeChangeCategory, + upgradePlanSeedDatasets, +} from './package-upgrade-plan.js'; type Fields = Record>; @@ -170,6 +177,20 @@ describe('seed datasets', () => { expect(plan.seeds.changed).toEqual([expect.objectContaining({ key: 'acme_deal', mode: 'upsert', fromMode: 'ignore', changed: ['mode'] })]); }); + it('the modes whose replay writes over a row it matches, of every mode the spec declares', () => { + expect(SeedMode.options.filter(seedModeOverwritesMatchedRows)).toEqual(['update', 'upsert']); + }); + + it('a dataset is keyed by its object, its non-default match key and locales, and its repeat', () => { + const keys = [...upgradePlanSeedDatasets({ data: [ + { object: 'acme_deal', records: [] }, + { object: 'acme_deal', externalId: 'code', records: [] }, + { object: 'acme_deal', locale: ['zh-CN', 'en'], records: [] }, + { object: 'acme_deal', records: [] }, + ] }).keys()]; + expect(keys).toEqual(['acme_deal', 'acme_deal by code', 'acme_deal [en,zh-CN]', 'acme_deal #2']); + }); + it('record key order is not a change', () => { const plan = comparePackageVersions( side({}, { data: [{ object: 'acme_deal', records: [{ name: 'a', label: 'A' }] }] }), diff --git a/packages/metadata-core/src/package-upgrade-plan.ts b/packages/metadata-core/src/package-upgrade-plan.ts index 27c0ccee74b..3236121f216 100644 --- a/packages/metadata-core/src/package-upgrade-plan.ts +++ b/packages/metadata-core/src/package-upgrade-plan.ts @@ -224,8 +224,9 @@ export function seedModeOverwritesMatchedRows(mode: string): boolean { return mode === 'upsert' || mode === 'update'; } -interface ReadDataset extends UpgradePlanSeedDataset { - raw: Bag; +/** A dataset as the plan reads it, with the declaration it was read from. */ +export interface UpgradePlanSeedDatasetRead extends UpgradePlanSeedDataset { + raw: Record; } function stableText(value: unknown): string { @@ -238,8 +239,14 @@ function stableText(value: unknown): string { } } -function datasetsOf(side: PackageUpgradeSide | undefined): Map { - const out = new Map(); +/** + * One side's seed datasets under the keys the plan names them by: `object`, + * plus its match key and its locales when they are not the defaults, and + * ` #n` on a second dataset under one key. Exported so a reader of the + * environment (`planUpgrade`) pairs datasets exactly as the plan does. + */ +export function upgradePlanSeedDatasets(side: PackageUpgradeSide | undefined): Map { + const out = new Map(); for (const item of side?.data ?? []) { if (!isBag(item) || typeof item.object !== 'string' || item.object === '') continue; const externalId = typeof item.externalId === 'string' || Array.isArray(item.externalId) @@ -259,13 +266,13 @@ function datasetsOf(side: PackageUpgradeSide | undefined): Map; + +/** In-memory driver: equality, `$in` and `$ne` WHERE; `find` honours `limit`. */ +function makeDriver(rowsByObject: Record, failing: ReadonlySet = new Set()) { + const matches = (row: Row, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k === '$and' && Array.isArray(v)) { + if (!v.every((w: any) => matches(row, w))) return false; + continue; + } + if (k.startsWith('$')) continue; + const value = row[k] === undefined ? null : row[k]; + if (v && typeof v === 'object' && Array.isArray((v as any).$in)) { + if (!(v as any).$in.includes(value)) return false; + continue; + } + if (v && typeof v === 'object' && '$ne' in (v as any)) { + if (value === (v as any).$ne) return false; + continue; + } + if (value !== (v === undefined ? null : v)) return false; + } + return true; + }; + const rowsOf = (object: string) => { + if (failing.has(object)) throw new Error(`connection lost reading ${object}`); + return (rowsByObject[object] ?? []).map((row) => ({ ...row })); + }; + return { + name: 'memory', version: '0.0.0', supports: {} as any, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find(object: string, ast: any) { + const rows = rowsOf(object).filter((row) => matches(row, ast?.where)); + return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; + }, + async findOne(object: string, ast: any) { return rowsOf(object).find((row) => matches(row, ast?.where)) ?? null; }, + async count(object: string, ast: any) { return rowsOf(object).filter((row) => matches(row, ast?.where)).length; }, + async create(_o: string, d: any) { return d; }, + async update(_o: string, _id: string, d: any) { return d; }, + async delete() { return true; }, + async bulkCreate(_o: string, r: any[]) { return r; }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + }; +} + +async function makeEngine(objects: Array<{ name: string; fields: Record }>, rows: Record, failing?: Set) { + const engine = new ObjectQL(); + engine.registerDriver(makeDriver(rows, failing) as any, true); + await engine.init(); + for (const object of objects) (engine.registry as any).registerObject(object); + return engine; +} + +const STAGE_FIELDS = { name: { type: 'text' }, label: { type: 'text' }, probability: { type: 'number' }, color: { type: 'text' } }; +const STAGE = { name: 'acme_stage', fields: STAGE_FIELDS }; + +const stages = (labels: Record) => + Object.entries(labels).map(([name, label]) => ({ name, label, probability: 10, color: 'grey' })); + +describe('rows a customer edited that the seed replay overwrites', () => { + it('a seeded upsert dataset with one row edited reports that row', async () => { + const installed = { objects: [STAGE], data: [{ object: 'acme_stage', mode: 'upsert', records: stages({ new: 'New', won: 'Won', lost: 'Lost' }) }] }; + // The new version relabels `lost`, and changes nothing a customer touched. + const incoming = { objects: [STAGE], data: [{ object: 'acme_stage', mode: 'upsert', records: stages({ new: 'New', won: 'Won', lost: 'Closed lost' }) }] }; + const engine = await makeEngine([STAGE], { + acme_stage: [ + { id: 's1', name: 'new', label: 'New', probability: 10, color: 'grey' }, + // The customer's edit, which the replay writes `Won` back over. + { id: 's2', name: 'won', label: 'Won (signed)', probability: 10, color: 'grey' }, + // As seeded: the replay's new label is an upgrade, not an erased edit. + { id: 's3', name: 'lost', label: 'Lost', probability: 10, color: 'grey' }, + ], + }); + + const { plan, environment } = await planPackageUpgrade(engine, installed, incoming, { getObject: (n) => engine.registry.getObject(n) }); + + expect(plan.seeds.changed.map((d) => d.key)).toEqual(['acme_stage']); + expect(environment.seeds).toEqual([ + { key: 'acme_stage', object: 'acme_stage', mode: 'upsert', changed: true, matchedRows: 3, editedRows: 1, editedKeys: ['won'] }, + ]); + }); + + it('an unchanged upsert dataset overwrites the same edits, and says it is unchanged', async () => { + const body = { objects: [STAGE], data: [{ object: 'acme_stage', mode: 'upsert', records: stages({ won: 'Won' }) }] }; + const engine = await makeEngine([STAGE], { acme_stage: [{ id: 's2', name: 'won', label: 'Signed', probability: 10, color: 'grey' }] }); + const { environment } = await planPackageUpgrade(engine, body, structuredClone(body)); + expect(environment.seeds).toEqual([expect.objectContaining({ changed: false, matchedRows: 1, editedRows: 1 })]); + }); + + it("an ignore dataset never writes over a matched row, so it is not read; nor is a column declared applyOn: 'create'", async () => { + const engine = await makeEngine([STAGE], { + acme_stage: [{ id: 's2', name: 'won', label: 'Won', probability: 90, color: 'grey' }], + acme_tier: [{ id: 't1', name: 'gold', label: 'Platinum' }], + }); + const { environment } = await planPackageUpgrade( + engine, + { data: [ + { object: 'acme_stage', mode: 'upsert', columns: { probability: { applyOn: 'create' } }, records: stages({ won: 'Won' }) }, + { object: 'acme_tier', records: [{ name: 'gold', label: 'Gold' }] }, + ] }, + { data: [ + { object: 'acme_stage', mode: 'upsert', columns: { probability: { applyOn: 'create' } }, records: stages({ won: 'Won' }) }, + { object: 'acme_tier', records: [{ name: 'gold', label: 'Gold tier' }] }, + ] }, + { getObject: (n) => engine.registry.getObject(n) }, + ); + expect(environment.seeds).toEqual([ + { key: 'acme_stage', object: 'acme_stage', mode: 'upsert', changed: false, matchedRows: 1, editedRows: 0, editedKeys: [] }, + ]); + }); + + it('a reading that fails is null, never zero, and is told', async () => { + const warn = vi.fn(); + const body = { objects: [STAGE], data: [{ object: 'acme_stage', mode: 'upsert', records: stages({ won: 'Won' }) }] }; + const engine = await makeEngine([STAGE], {}, new Set(['acme_stage'])); + const { environment } = await planPackageUpgrade(engine, body, structuredClone(body), { warn }); + expect(environment.seeds).toEqual([expect.objectContaining({ matchedRows: null, editedRows: null })]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('answered null, never zero'), expect.objectContaining({ object: 'acme_stage' })); + }); +}); + +describe('rows in each column the upgrade touches', () => { + const lead = (fields: Record) => ({ name: 'acme_lead', fields }); + + it('removed, suspected-rename, renamed and retyped columns, each with its rows; a removed object with its rows', async () => { + const before = lead({ + company: { type: 'text' }, legacy: { type: 'text' }, amount: { type: 'text' }, total: { type: 'formula' }, region: { type: 'select' }, + }); + const after = lead({ + account_name: { type: 'text' }, amount_value: { type: 'number', renamedFrom: ['amount'] }, total: { type: 'number' }, area: { type: 'select', renamedFrom: ['region'] }, + }); + // The registry holds the installed object's stored columns (the computed + // `total` and the option-less `region` would only add boot-log noise here). + const registered = lead({ company: { type: 'text' }, legacy: { type: 'text' }, amount: { type: 'text' }, region: { type: 'text' } }); + const engine = await makeEngine([registered, { name: 'acme_old', fields: { x: { type: 'text' } } }], { + acme_lead: [ + { id: 'l1', company: 'Acme', legacy: null, amount: '10', region: 'emea' }, + { id: 'l2', company: 'Globex', legacy: 'x', amount: null, region: null }, + { id: 'l3', company: null, legacy: null, amount: '7', region: 'apac' }, + ], + acme_old: [{ id: 'o1', x: 'a' }, { id: 'o2', x: 'b' }], + }); + + const { environment } = await planPackageUpgrade( + engine, + { objects: [before, { name: 'acme_old', fields: { x: { type: 'text' } } }] }, + { objects: [after] }, + ); + + expect(environment.columns).toEqual([ + { object: 'acme_lead', field: 'company', reasons: ['removed', 'suspected_rename'], rows: 2 }, + { object: 'acme_lead', field: 'legacy', reasons: ['removed', 'suspected_rename'], rows: 1 }, + { object: 'acme_lead', field: 'amount', reasons: ['renamed', 'type_change'], rows: 2 }, + { object: 'acme_lead', field: 'region', reasons: ['renamed'], rows: 2 }, + ]); + // `total` was computed (formula → number is `safe`): no stored column to count. + expect(environment.removedObjects).toEqual([{ object: 'acme_old', rows: 2 }]); + }); + + it('an identical pair reads nothing', async () => { + const engine = await makeEngine([lead({ company: { type: 'text' } })], { acme_lead: [{ id: 'l1', company: 'Acme' }] }); + const body = { objects: [lead({ company: { type: 'text' } })] }; + expect((await planPackageUpgrade(engine, body, structuredClone(body))).environment).toEqual({ columns: [], removedObjects: [], seeds: [] }); + }); +}); + +describe("the `manifest` service's planUpgrade", () => { + it('folds a multi-package artifact with the platform reader and reads through the engine', async () => { + const kernel = new ObjectKernel({ logger: { level: 'silent' }, gracefulShutdown: false }); + await kernel.use(new ObjectQLPlugin()); + await kernel.bootstrap(); + try { + const ql = kernel.getService('objectql'); + ql.registerDriver(makeDriver({ acme_lead: [{ id: 'l1', company: 'Acme' }, { id: 'l2', company: null }] }) as any, true); + (ql.registry as any).registerObject({ name: 'acme_lead', fields: { company: { type: 'text' } } }); + const artifact = (fields: Record) => ({ + packages: [{ manifest: { id: 'com.acme.crm', name: 'crm', version: '1.0.0', type: 'app', objects: [{ name: 'acme_lead', fields }] } }], + }); + + const manifest = kernel.getService<{ planUpgrade(a: unknown, b: unknown): ReturnType }>('manifest'); + const { plan, environment } = await manifest.planUpgrade(artifact({ company: { type: 'text' } }), artifact({ account_name: { type: 'text' } })); + + expect(plan.suspectedRenames.map((s) => `${s.field}→${s.candidates.map((c) => c.field).join('|')}`)).toEqual(['company→account_name']); + expect(environment.columns).toEqual([{ object: 'acme_lead', field: 'company', reasons: ['removed', 'suspected_rename'], rows: 1 }]); + } finally { + await kernel.shutdown(); + } + }); +}); diff --git a/packages/objectql/src/package-upgrade-environment.ts b/packages/objectql/src/package-upgrade-environment.ts new file mode 100644 index 00000000000..f03342c07ce --- /dev/null +++ b/packages/objectql/src/package-upgrade-environment.ts @@ -0,0 +1,390 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The environment half of the package upgrade plan (epic cloud#3652 Q3, + * objectstack#30884): what only the running environment knows about an + * upgrade from the installed version of a package to an incoming one. + * + * The comparison itself is `comparePackageVersions` + * (`@objectstack/metadata-core`), pure and shared with `os diff`. This module + * adds two readings of the data the installed version left behind: + * + * - **Rows in a touched column.** For every stored column the upgrade + * removes, renames (declared or suspected) or retypes, the rows that hold a + * value in it, and for every object it removes, the rows it holds. Upgrade + * schema sync is additive (`schema-drift.ts`): none of these columns is + * dropped or carried, so these are the rows a rename leaves behind and a + * type change keeps under a declaration of another type. + * - **Rows a customer edited that the seed replay overwrites.** An upgrade + * replays every dataset of the incoming version. A dataset whose mode writes + * over a row it matches (`seedModeOverwritesMatchedRows`: `upsert`, + * `update`) puts the incoming values back over every row it matches. A row + * whose current value differs from the value the INSTALLED version seeded, + * in a column the replay writes with another value, is a customer's edit + * the replay erases. Counted for every such dataset the installed version + * also declares, changed or not: an unchanged `upsert` dataset overwrites + * the same edits. + * + * Read-only: every read is a `count` or a `find` with the system context, across + * every organization (a package upgrade replaces the package for all of them). + * A reading that fails is answered `null` and never taken for zero. + * + * The comparison of a seed value with a stored one follows the replay's own + * (`SeedLoaderService.isNoOpReplay`, `@objectstack/metadata-protocol`): a + * temporal column in its stored form and a seed blank in the form the write + * door stores it, through the same `@objectstack/core` rules; a credential, a + * file, a reference (resolved from a natural key on write) and an expression + * (evaluated on each load) are never compared, so they count no row. + */ + +import { + blankAsNullColumns, + blankStorageForm, + resolveStackCollection, + temporalComparandKind, + temporalStorageForm, + type BlankAsNullReach, + type TemporalComparandKind, +} from '@objectstack/core'; +import { + comparePackageVersions, + seedModeOverwritesMatchedRows, + upgradePlanSeedDatasets, + type PackageUpgradePlan, + type PackageUpgradeSide, +} from '@objectstack/metadata-core'; +import { FILE_REFERENCE_TYPES, isMaskedOnReadFieldType, isVirtualSearchField } from '@objectstack/spec/data'; + +/** Why a column is read: what the upgrade does to it. */ +export type UpgradeColumnReason = 'removed' | 'renamed' | 'suspected_rename' | 'type_change'; + +/** Rows holding a value in one installed column the upgrade touches. */ +export interface UpgradeColumnRows { + object: string; + /** The INSTALLED key: the column the rows are in. */ + field: string; + reasons: UpgradeColumnReason[]; + /** Rows whose value is not null; `null` when the reading failed. */ + rows: number | null; +} + +/** Rows of an object the upgrade removes. */ +export interface UpgradeObjectRows { + object: string; + /** `null` when the reading failed. */ + rows: number | null; +} + +/** One overwriting dataset the installed version also declares. */ +export interface UpgradeSeedOverwrite { + key: string; + object: string; + /** The incoming mode (the one the replay runs). */ + mode: string; + /** Does the plan list the dataset as changed? */ + changed: boolean; + /** Rows the replay matches by `externalId`; `null` when the reading failed. */ + matchedRows: number | null; + /** Of those, rows a customer edited that the replay overwrites; `null` when the reading failed. */ + editedRows: number | null; + /** The natural keys of the first edited rows ({@link EDITED_KEYS_SHOWN} at most). */ + editedKeys: string[]; +} + +export interface PackageUpgradeEnvironment { + columns: UpgradeColumnRows[]; + removedObjects: UpgradeObjectRows[]; + seeds: UpgradeSeedOverwrite[]; +} + +/** The comparison and the environment's readings of it, as `planUpgrade` answers. */ +export interface PackageUpgradePlanReading { + plan: PackageUpgradePlan; + environment: PackageUpgradeEnvironment; +} + +/** The engine reads this module takes. `ObjectQL` satisfies it. */ +export interface UpgradeEnvironmentEngine { + count(object: string, query?: any, options?: any): Promise; + find(object: string, query?: any, options?: any): Promise; +} + +export interface PlanPackageUpgradeOptions { + /** The object definition the environment serves now (the installed version's). */ + getObject?: (name: string) => unknown; + /** Told once per failed reading. */ + warn?: (message: string, meta?: Record) => void; +} + +/** How many edited rows an answer names by key. */ +export const EDITED_KEYS_SHOWN = 20; + +const SYSTEM_READ = { context: { isSystem: true } } as const; +/** Columns a seed names by a natural key that the write resolves to an id. */ +const REFERENCE_TYPES: ReadonlySet = new Set(['lookup', 'master_detail', 'user']); + +type Bag = Record; +const isBag = (value: unknown): value is Bag => typeof value === 'object' && value !== null && !Array.isArray(value); + +/** One side as the comparison reads it: a multi-package artifact folded by the platform's one reader. */ +function sideOf(artifact: unknown): PackageUpgradeSide { + return { objects: resolveStackCollection(artifact, 'objects'), data: resolveStackCollection(artifact, 'data') }; +} + +/** + * Compare the installed version of a package with the incoming one and read + * the environment for what the comparison touches. Both artifacts are taken + * as the registry reads them (a caller holding a published artifact converts + * it forward first, as it does to register it). + */ +export async function planPackageUpgrade( + engine: UpgradeEnvironmentEngine, + installed: unknown, + incoming: unknown, + options: PlanPackageUpgradeOptions = {}, +): Promise { + const installedSide = sideOf(installed); + const incomingSide = sideOf(incoming); + const plan = comparePackageVersions(installedSide, incomingSide); + const environment: PackageUpgradeEnvironment = { + columns: await readTouchedColumns(engine, plan, options), + removedObjects: [], + seeds: await readSeedOverwrites(engine, plan, installedSide, incomingSide, options), + }; + for (const object of plan.objects.removed) { + environment.removedObjects.push({ object, rows: await reading(options, { object }, () => engine.count(object, {}, SYSTEM_READ)) }); + } + return { plan, environment }; +} + +async function reading(options: PlanPackageUpgradeOptions, meta: Record, read: () => Promise): Promise { + try { + return await read(); + } catch (error) { + options.warn?.('[planUpgrade] a reading of the installed data failed; it is answered null, never zero', { + ...meta, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +async function readTouchedColumns( + engine: UpgradeEnvironmentEngine, + plan: PackageUpgradePlan, + options: PlanPackageUpgradeOptions, +): Promise { + const touched = new Map(); + const touch = (object: string, field: string, reason: UpgradeColumnReason) => { + const id = `${object}\u0000${field}`; + const entry = touched.get(id) ?? { object, field, reasons: [], rows: null }; + if (!entry.reasons.includes(reason)) entry.reasons.push(reason); + touched.set(id, entry); + }; + // A removed computed field held no column: nothing to count. + for (const f of plan.fields.removed) if (f.stored) touch(f.object, f.field, 'removed'); + for (const r of plan.declaredRenames) touch(r.object, r.from, 'renamed'); + for (const s of plan.suspectedRenames) touch(s.object, s.field, 'suspected_rename'); + for (const t of plan.typeChanges) { + // Retyped from a computed field: the installed version stored nothing. + if (t.category === 'safe') continue; + touch(t.object, t.renamedFrom ?? t.field, 'type_change'); + } + const out = [...touched.values()]; + for (const entry of out) { + entry.rows = await reading(options, { object: entry.object, field: entry.field }, () => + engine.count(entry.object, { where: { [entry.field]: { $ne: null } } }, SYSTEM_READ)); + } + return out; +} + +// ── seeds ─────────────────────────────────────────────────────────────── + +interface Dataset { + key: string; + object: string; + externalId: string | string[]; + mode: string; + records: Bag[]; + createOnly: Set; +} + +/** One side's datasets, paired by the plan's own keys (`upgradePlanSeedDatasets`). */ +function datasetsByKey(side: PackageUpgradeSide): Map { + const out = new Map(); + for (const [key, read] of upgradePlanSeedDatasets(side)) { + const keyFields = new Set(Array.isArray(read.externalId) ? read.externalId : [read.externalId]); + // `applyOn: 'create'` takes a column out of every write to a matched row + // (the loader's `createOnlySeedColumns`); a match-key column stays in. + const createOnly = new Set(); + if (isBag(read.raw.columns)) { + for (const [column, policy] of Object.entries(read.raw.columns)) { + if (isBag(policy) && policy.applyOn === 'create' && !keyFields.has(column)) createOnly.add(column); + } + } + out.set(key, { + key, + object: read.object, + externalId: read.externalId, + mode: read.mode, + records: Array.isArray(read.raw.records) ? read.raw.records.filter(isBag) : [], + createOnly, + }); + } + return out; +} + +/** The natural key of a record or row, `''` when a part is missing (the loader's `externalIdKey`). */ +function naturalKey(record: Bag, externalId: string | string[]): string { + const parts: string[] = []; + for (const field of Array.isArray(externalId) ? externalId : [externalId]) { + const value = record[field]; + if (value === undefined || value === null || value === '') return ''; + parts.push(String(value)); + } + return parts.join('\u0000'); +} + +/** A seed value evaluated on each load (`cel\`…\``), whose value no stored one is compared with. */ +function isExpressionValue(value: unknown): boolean { + return isBag(value) && typeof value.dialect === 'string'; +} + +interface Comparands { + temporal: Map; + blankAsNull: ReadonlyMap; + /** Columns never compared: computed, credential, file, reference. */ + skipped: Set; +} + +function comparandsOf(definition: unknown): Comparands { + const object = isBag(definition) ? definition : undefined; + const fields = isBag(object?.fields) ? object!.fields as Record : {}; + const temporal = new Map(); + const skipped = new Set(); + for (const [name, def] of Object.entries(fields)) { + const type = def?.type; + const kind = temporalComparandKind(type); + if (kind) temporal.set(name, kind); + if (typeof type !== 'string') continue; + if ( + isVirtualSearchField({ type } as Parameters[0]) + || isMaskedOnReadFieldType(type, object?.managedBy) + || FILE_REFERENCE_TYPES.has(type) + || REFERENCE_TYPES.has(type) + ) skipped.add(name); + } + return { temporal, blankAsNull: blankAsNullColumns(object as Parameters[0]), skipped }; +} + +/** Would the replay's comparison call these the same stored value? (the loader's `seedValueEquals`) */ +function sameStoredValue(seed: unknown, stored: unknown, kind: TemporalComparandKind | undefined): boolean { + if (seed === stored) return true; + if (seed == null || stored == null) return seed == null && stored == null; + if (kind) { + const form = temporalStorageForm(seed, kind); + return typeof form === 'string' && form === temporalStorageForm(stored, kind); + } + if (typeof seed === 'boolean' || typeof stored === 'boolean') { + const toNum = (v: unknown) => (typeof v === 'boolean' ? Number(v) : Number(String(v))); + return toNum(seed) === toNum(stored); + } + if (seed instanceof Date || stored instanceof Date) { + const toTime = (v: unknown) => (v instanceof Date ? v.getTime() : typeof v === 'number' ? v : Date.parse(String(v))); + const a = toTime(seed); + const b = toTime(stored); + return Number.isFinite(a) && Number.isFinite(b) && a === b; + } + if (typeof seed === 'object' || typeof stored === 'object') { + try { + return JSON.stringify(seed) === JSON.stringify(stored); + } catch { + return false; + } + } + return String(seed) === String(stored); +} + +/** + * Is `row` a customer's edit the replay of `incoming` erases? Some column the + * replay writes holds a value other than the one `installed` seeded, and the + * replay writes another value still. + */ +function isOverwrittenEdit(row: Bag, installed: Bag, incoming: Bag, dataset: Dataset, comparands: Comparands): boolean { + const keyFields = new Set(Array.isArray(dataset.externalId) ? dataset.externalId : [dataset.externalId]); + for (const [column, value] of Object.entries(incoming)) { + if (column === 'id' || keyFields.has(column) || dataset.createOnly.has(column) || comparands.skipped.has(column)) continue; + if (!(column in installed)) continue; + if (isExpressionValue(value) || isExpressionValue(installed[column])) continue; + const kind = comparands.temporal.get(column); + const reach = comparands.blankAsNull.get(column); + const stored = row[column]; + const seededAsInstalled = sameStoredValue(blankStorageForm(installed[column], reach), stored, kind); + const replayKeeps = sameStoredValue(blankStorageForm(value, reach), stored, kind); + if (!seededAsInstalled && !replayKeeps) return true; + } + return false; +} + +async function readSeedOverwrites( + engine: UpgradeEnvironmentEngine, + plan: PackageUpgradePlan, + installedSide: PackageUpgradeSide, + incomingSide: PackageUpgradeSide, + options: PlanPackageUpgradeOptions, +): Promise { + const before = datasetsByKey(installedSide); + const after = datasetsByKey(incomingSide); + const changedKeys = new Set(plan.seeds.changed.map((d) => d.key)); + const out: UpgradeSeedOverwrite[] = []; + for (const [key, dataset] of after) { + const prior = before.get(key); + if (!prior || !seedModeOverwritesMatchedRows(dataset.mode)) continue; + const entry: UpgradeSeedOverwrite = { + key, object: dataset.object, mode: dataset.mode, changed: changedKeys.has(key), matchedRows: null, editedRows: null, editedKeys: [], + }; + out.push(entry); + + const installedByKey = new Map(); + for (const record of prior.records) { + const k = naturalKey(record, prior.externalId); + if (k !== '') installedByKey.set(k, record); + } + const incomingByKey = new Map(); + for (const record of dataset.records) { + const k = naturalKey(record, dataset.externalId); + if (k !== '') incomingByKey.set(k, record); + } + if (incomingByKey.size === 0) { + entry.matchedRows = 0; + entry.editedRows = 0; + continue; + } + const first = Array.isArray(dataset.externalId) ? dataset.externalId[0] : dataset.externalId; + const firstValues = [...new Set(dataset.records.map((r) => r[first]).filter((v) => v !== undefined && v !== null && v !== ''))]; + const rows = await reading(options, { object: dataset.object, dataset: key }, () => + engine.find(dataset.object, { where: { [first]: { $in: firstValues } } }, SYSTEM_READ)); + if (rows === null) continue; + + const comparands = comparandsOf(options.getObject?.(dataset.object)); + let matched = 0; + let edited = 0; + for (const row of rows) { + if (!isBag(row)) continue; + const k = naturalKey(row, dataset.externalId); + const incomingRecord = incomingByKey.get(k); + if (!incomingRecord) continue; + matched++; + const installedRecord = installedByKey.get(k); + // A row the installed version did not seed has no seeded value to have been edited from. + if (!installedRecord) continue; + if (isOverwrittenEdit(row, installedRecord, incomingRecord, dataset, comparands)) { + edited++; + if (entry.editedKeys.length < EDITED_KEYS_SHOWN) entry.editedKeys.push(k.split('\u0000').join('+')); + } + } + entry.matchedRows = matched; + entry.editedRows = edited; + } + return out; +} diff --git a/packages/objectql/src/plugin.ts b/packages/objectql/src/plugin.ts index a5e31f6caf9..d4133a9efc5 100644 --- a/packages/objectql/src/plugin.ts +++ b/packages/objectql/src/plugin.ts @@ -69,6 +69,7 @@ import { SecurityCatalogNameConflictError, findEnvironmentHeldSecurityCatalogNam // boot's seal and its one switch (`bare-name-one-holder.ts`, "The cold boot"). import { sealBareNameHoldersAtColdBoot } from './registry.js'; import { BARE_NAME_COLD_BOOT_MODE } from './bare-name-one-holder.js'; +import { planPackageUpgrade } from './package-upgrade-environment.js'; // [ADR-0029 D3] The registry's own refusal of a second owner — read, never // re-derived, when a residual object's name is already owned. import { ObjectOwnershipConflictError, collectManifestBoundActions, collectManifestObjectNames, describeBoundActionCollision, describeUnboundActionTargets, FIND_BOUND_ACTION_COLLISIONS, type BoundActionCollision } from './registry.js'; @@ -1030,6 +1031,16 @@ export class ObjectQLPlugin implements Plugin { // {@link restoreExtendedObjectCopies}. restoreExtendedObjectCopies: (packageId: string): Promise => this.restoreExtendedObjectCopies(ctx, packageId), + // [epic cloud#3652 Q3, #30884] The package upgrade plan with what only this + // environment knows: rows in each column the upgrade touches and rows a + // customer edited that the seed replay overwrites. Read-only; the install + // door's dry run and a host's install door call it. See + // {@link planPackageUpgrade}. + planUpgrade: (installed: unknown, incoming: unknown) => + planPackageUpgrade(ql, installed, incoming, { + getObject: (name) => ql.registry.getObject(name), + warn: (message, meta) => ctx.logger.warn(message, meta), + }), }); ctx.logger.info('ObjectQL engine registered', { From 237d6c6c411f501a203601a0c593b983ecd4aa99 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:21:37 +0000 Subject: [PATCH 5/7] feat(cloud-connection): install-local dryRun answers the upgrade plan and writes nothing `POST /api/v1/marketplace/install-local` with `dryRun: true` answers 200 with the `manifest` service's `planUpgrade` of the installed version (its ledger entry, converted forward) against the incoming one, after every refusal that says the package cannot install at all and before the first confirmation ceremony or write. A `dryRun` that is not a boolean, a dry run of an `unmanaged` install and one of a customization pair are refused 400; a runtime whose `manifest` service answers no `planUpgrade` answers 503. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- .../src/marketplace-install-local-plugin.ts | 126 +++++++++++ ...etplace-install-local-upgrade-plan.test.ts | 206 ++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 packages/cloud-connection/src/marketplace-install-local-upgrade-plan.test.ts diff --git a/packages/cloud-connection/src/marketplace-install-local-plugin.ts b/packages/cloud-connection/src/marketplace-install-local-plugin.ts index f5d54196187..975487202cf 100644 --- a/packages/cloud-connection/src/marketplace-install-local-plugin.ts +++ b/packages/cloud-connection/src/marketplace-install-local-plugin.ts @@ -95,6 +95,18 @@ * answers `503 SERVICE_UNAVAILABLE`. Under a wall only the operator * reaches this step (ADR-0131 D12 items 7–8). * + * [epic cloud#3652 Q3, #30884] `dryRun: true` answers `200` with the + * package upgrade plan — the installed version (its ledger entry) + * against this one: objects, fields, type changes, declared and + * suspected renames, seed datasets — and the `manifest` service's + * readings of this environment (rows in each touched column, rows a + * customer edited that the seed replay overwrites), after every + * refusal that says the package cannot install at all and before + * any confirmation ceremony or write. Nothing is installed or + * written. A dry run of an `unmanaged` install or of a + * customization pair answers `400 INVALID_REQUEST`, as does a + * `dryRun` that is not a boolean. + * * GET /api/v1/marketplace/install-local * → lists currently installed marketplace packages. Requires an * authenticated principal (anonymous → 401); `installedBy` and @@ -1800,6 +1812,35 @@ export class MarketplaceInstallLocalPlugin implements Plugin { } const installMode: InstallMode = requestedMode.data; + // 0'. [epic cloud#3652 Q3, #30884] ⭐ `dryRun: true` answers the + // package upgrade plan and writes nothing (step 2a′ below). Read + // here with the mode, before anything is fetched: a key this door + // cannot read is refused, never read as "install for real", and a + // dry run of an `unmanaged` install (a copy, which follows no + // upgrade, and whose branch below writes) is refused rather than + // run. + if (body?.dryRun !== undefined && typeof body.dryRun !== 'boolean') { + return c.json({ + success: false, + error: { + code: 'INVALID_REQUEST', + message: `\`dryRun\` must be true or false, received ${JSON.stringify(body.dryRun)}. Nothing was installed or written.`, + }, + }, 400); + } + const dryRun = body?.dryRun === true; + if (dryRun && installMode === 'unmanaged') { + return c.json({ + success: false, + error: { + code: 'INVALID_REQUEST', + message: 'A dry run previews a managed install or upgrade; an `unmanaged` install is a copy that follows no ' + + 'upgrade, so there is no plan to preview. Omit `dryRun`, or omit `installMode`. Nothing was installed ' + + 'or written.', + }, + }, 400); + } + // ── Offline path: an inline manifest was supplied (file import). ── // Bypass the cloud-fetch entirely; no OS_CLOUD_URL required. const inlineManifest = body?.manifest && typeof body.manifest === 'object' ? body.manifest : null; @@ -2229,6 +2270,19 @@ export class MarketplaceInstallLocalPlugin implements Plugin { }, UNRUNNABLE_REFUSAL_STATUS); } if (pairBaseId !== null) { + // [#30884] The pair's branch registers and writes on its own path; + // the upgrade plan previews a package's own data schema, not a + // pair's, so a dry run of it is refused rather than run. + if (dryRun) { + return c.json({ + success: false, + error: { + code: 'INVALID_REQUEST', + message: `Package '${manifestId}' is a customization pair over '${pairBaseId}': a dry run previews ` + + 'a package install or upgrade, not a pair. Omit `dryRun`. Nothing was installed or written.', + }, + }, 400); + } return this.installCustomizationPair(c, ctx, { manifest, manifestId, @@ -2325,6 +2379,15 @@ export class MarketplaceInstallLocalPlugin implements Plugin { }, 409); } + // 2a′. [epic cloud#3652 Q3, #30884] ⭐ THE DRY RUN answers here, + // after every refusal that says this package cannot install at + // all (payload, id, wall, handshake, dependencies, mode, code it + // cannot run, namespace, conflict, datasource holder) and before + // the first confirmation ceremony (2b) and the first write (3): + // the plan is what an installer reads before deciding, so it is + // answered whether or not a ceremony would stop the install. + if (dryRun) return this.answerUpgradeDryRun(c, ctx, { manifest, manifestId, version }); + // 2b. [ADR-0120 D5e] `isolated`-posture gate on installation-wide // uniques. Runs BEFORE hot-register and before anything is written // to the ledger: a stopped install must leave the runtime exactly as @@ -4835,6 +4898,69 @@ export class MarketplaceInstallLocalPlugin implements Plugin { * * An engine without that door (not ObjectQL) is handed the body as before. */ + /** + * [epic cloud#3652 Q3, #30884] The dry run (`dryRun: true`, step 2a′): the + * package upgrade plan of the installed version against this one, with + * what only this environment knows, and nothing installed or written. + * + * Both versions are compared as the registry reads them — each converted + * forward ({@link convertPackageBodyForward}), the installed one from its + * ledger entry — by the `manifest` service's `planUpgrade` + * (`@objectstack/objectql`): the comparison is + * `comparePackageVersions` (`@objectstack/metadata-core`), the one + * `os diff` reads, and the readings are rows in each column the upgrade + * touches and rows a customer edited that the seed replay overwrites. + * + * A runtime whose `manifest` service answers no `planUpgrade` answers 503: + * a preview that could not run is never answered as an empty one. A + * ledger entry that will not parse is said so, and the plan is then + * against no installed version. + */ + private answerUpgradeDryRun = async ( + c: any, + ctx: PluginContext, + args: { manifest: any; manifestId: string; version: string }, + ): Promise => { + const { manifest, manifestId, version } = args; + let service: { planUpgrade?: (installed: unknown, incoming: unknown) => Promise } | undefined; + try { service = ctx.getService('manifest') as typeof service; } catch { /* no manifest service */ } + if (typeof service?.planUpgrade !== 'function') { + return c.json({ + success: false, + error: { + code: 'SERVICE_UNAVAILABLE', + message: `A dry run of '${manifestId}' cannot be answered: this runtime's \`manifest\` service does not ` + + 'answer `planUpgrade`, which the plan is read from. Nothing was installed or written.', + }, + }, 503); + } + const ledgerRead = this.ledger.read(manifestId); + const installedEntry = ledgerRead.entry; + const installed = installedEntry?.manifest ? this.convertPackageBodyForward(ctx, installedEntry.manifest) : undefined; + const reading = await service.planUpgrade(installed, this.convertPackageBodyForward(ctx, manifest)) as { + plan: unknown; + environment: unknown; + }; + return c.json({ + success: true, + data: { + dryRun: true, + manifestId, + version, + installedVersion: installedEntry?.version ?? null, + ...(ledgerRead.failure ? { installedRecordUnreadable: true } : {}), + plan: reading.plan, + environment: reading.environment, + message: installedEntry + ? `Dry run: the plan of upgrading '${manifestId}' from ${installedEntry.version} to ${version}. Nothing was installed or written.` + : ledgerRead.failure + ? `Dry run: the record of the installed '${manifestId}' could not be read, so the plan lists everything ` + + `${version} adds, against no installed version. Nothing was installed or written.` + : `Dry run: '${manifestId}' is not installed here, so the plan lists everything ${version} adds. Nothing was installed or written.`, + }, + }, 200); + }; + private convertPackageBodyForward = (ctx: PluginContext, manifest: unknown): any => { let ql: { convertPackageBodyForward?: (body: unknown) => unknown } | undefined; try { ql = ctx.getService('objectql') as typeof ql; } catch { /* no data engine */ } diff --git a/packages/cloud-connection/src/marketplace-install-local-upgrade-plan.test.ts b/packages/cloud-connection/src/marketplace-install-local-upgrade-plan.test.ts new file mode 100644 index 00000000000..3946df319d3 --- /dev/null +++ b/packages/cloud-connection/src/marketplace-install-local-upgrade-plan.test.ts @@ -0,0 +1,206 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [epic cloud#3652 Q3, #30884] The marketplace door's dry run: `dryRun: true` + * answers the package upgrade plan of the installed version (its ledger + * entry) against the incoming one, read from the `manifest` service's + * `planUpgrade`, and writes nothing — no registration, no schema sync, no + * ledger change. + * + * `planUpgrade` here is the comparison the real service runs + * (`comparePackageVersions`, each side folded by `resolveStackCollection`) + * with an empty environment reading: the readings themselves are pinned on + * the real engine in `@objectstack/objectql`'s + * `package-upgrade-environment.test.ts`. What this file pins is the door: what + * it hands the service, what it answers, and that it writes nothing. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +// [#21321] An install binds the package's handlers through +// `@objectstack/runtime` (a lazy `import()` inside the plugin). Its first load +// is paid here, at module top — never inside a clocked `it`. +import '@objectstack/runtime'; +import { resolveStackCollection } from '@objectstack/core'; +import { comparePackageVersions } from '@objectstack/metadata-core'; +import { MarketplaceInstallLocalPlugin } from './marketplace-install-local-plugin.js'; +import { installerAuthService, withInstallerGrants } from './install-local-principal.fixtures.js'; +import { LocalManifestSource } from './local-manifest-source.js'; + +type Handler = (c: any) => Promise; +type Row = Record; + +const VENDOR = 'com.acme.leads'; +const INSTALLER = 'usr_installer'; + +function vendorBody(version: string, fields: Row, data: Row[] = []): Row { + return { + id: VENDOR, + namespace: 'acme', + version, + objects: [{ name: 'acme_lead', label: 'Lead', fields }], + ...(data.length > 0 ? { data } : {}), + }; +} + +const V1 = vendorBody('1.0.0', { company: { type: 'text' } }, [{ object: 'acme_lead', mode: 'upsert', records: [{ name: 'a', company: 'Acme' }] }]); +const V2 = vendorBody('2.0.0', { account_name: { type: 'text' } }, [{ object: 'acme_lead', mode: 'upsert', records: [{ name: 'a', account_name: 'Acme' }] }]); + +function makeRawApp() { + const routes = new Map(); + return { + routes, + get: (p: string, h: Handler) => routes.set(`GET ${p}`, h), + post: (p: string, h: Handler) => routes.set(`POST ${p}`, h), + delete: (p: string, h: Handler) => routes.set(`DELETE ${p}`, h), + }; +} + +function makeC(body: any) { + const json = vi.fn((payload: any, status?: number) => ({ payload, status: status ?? 200 })); + return { + req: { + url: 'http://localhost:3000/api/v1/marketplace/install-local', + raw: new Request('http://localhost:3000/x'), + json: async () => body, + param: () => undefined, + }, + json, + }; +} + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'mil-upgrade-plan-')); +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +const fold = (artifact: unknown) => ({ objects: resolveStackCollection(artifact, 'objects'), data: resolveStackCollection(artifact, 'data') }); + +async function mount(options: { installed?: Row; planUpgrade?: false } = {}) { + const register = vi.fn(); + const syncSchemas = vi.fn(async () => undefined); + const planUpgrade = vi.fn(async (installed: unknown, incoming: unknown) => ({ + plan: comparePackageVersions(fold(installed), fold(incoming)), + environment: { columns: [], removedObjects: [], seeds: [] }, + })); + const rawApp = makeRawApp(); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; + const services: Record = { + manifest: options.planUpgrade === false ? { register } : { register, planUpgrade }, + auth: installerAuthService(INSTALLER), + objectql: withInstallerGrants({ syncSchemas }, INSTALLER), + }; + const hooks = new Map(); + const ctx = { + hook: (e: string, h: any) => hooks.set(e, h), + getService: (name: string) => { + if (name === 'http-server') return { getRawApp: () => rawApp }; + if (!(name in services)) throw new Error(`no ${name}`); + return services[name]; + }, + logger, + }; + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx as any); + await hooks.get('kernel:ready')?.(); + const install = rawApp.routes.get('POST /api/v1/marketplace/install-local')!; + if (options.installed) { + const first = await install(makeC({ manifest: options.installed })); + expect(first.status).toBe(200); + } + register.mockClear(); + syncSchemas.mockClear(); + return { install, register, syncSchemas, planUpgrade }; +} + +/** Every ledger file and its bytes. */ +const ledgerFiles = (): Record => + Object.fromEntries(readdirSync(dir).sort().map((f) => [f, readFileSync(join(dir, f), 'utf8')])); + +describe('dryRun: true answers the upgrade plan and writes nothing', () => { + it('a re-install: the installed ledger body against the incoming one; registry, schema and ledger untouched', async () => { + const door = await mount({ installed: V1 }); + const before = ledgerFiles(); + + const res = await door.install(makeC({ manifest: V2, dryRun: true })); + + expect(res.status).toBe(200); + expect(res.payload.data).toMatchObject({ dryRun: true, manifestId: VENDOR, version: '2.0.0', installedVersion: '1.0.0' }); + expect(res.payload.data.plan.suspectedRenames).toEqual([ + { object: 'acme_lead', field: 'company', type: 'text', candidates: [{ field: 'account_name', type: 'text' }] }, + ]); + expect(res.payload.data.plan.seeds.changed).toEqual([expect.objectContaining({ key: 'acme_lead', mode: 'upsert', changed: ['records'] })]); + expect(res.payload.data.environment).toEqual({ columns: [], removedObjects: [], seeds: [] }); + expect(res.payload.data.message).toMatch(/from 1\.0\.0 to 2\.0\.0\. Nothing was installed or written\./); + + // The service was handed the installed version from the ledger and the incoming one. + const [installed, incoming] = door.planUpgrade.mock.calls[0]; + expect(resolveStackCollection(installed, 'objects')).toEqual(V1.objects); + expect(resolveStackCollection(incoming, 'objects')).toEqual(V2.objects); + + // Nothing written: no registration, no schema sync, the ledger byte for byte. + expect(door.register).not.toHaveBeenCalled(); + expect(door.syncSchemas).not.toHaveBeenCalled(); + expect(ledgerFiles()).toEqual(before); + expect(new LocalManifestSource(dir).read(VENDOR).entry?.version).toBe('1.0.0'); + }); + + it('a package not installed here: the plan lists what it adds, and no ledger entry appears', async () => { + const door = await mount(); + const res = await door.install(makeC({ manifest: V1, dryRun: true })); + expect(res.status).toBe(200); + expect(res.payload.data).toMatchObject({ dryRun: true, installedVersion: null }); + expect(res.payload.data.plan.objects.added).toEqual(['acme_lead']); + expect(res.payload.data.message).toMatch(/is not installed here/); + expect(door.register).not.toHaveBeenCalled(); + expect(door.syncSchemas).not.toHaveBeenCalled(); + expect(ledgerFiles()).toEqual({}); + }); + + it('CONTROL: the same request without dryRun installs', async () => { + const door = await mount({ installed: V1 }); + const res = await door.install(makeC({ manifest: V2 })); + expect(res.status).toBe(200); + expect(door.register).toHaveBeenCalledTimes(1); + expect(door.planUpgrade).not.toHaveBeenCalled(); + expect(new LocalManifestSource(dir).read(VENDOR).entry?.version).toBe('2.0.0'); + }); + + it('a runtime whose manifest service answers no planUpgrade answers 503, never an empty plan', async () => { + const door = await mount({ installed: V1, planUpgrade: false }); + const before = ledgerFiles(); + const res = await door.install(makeC({ manifest: V2, dryRun: true })); + expect(res.status).toBe(503); + expect(res.payload.error.code).toBe('SERVICE_UNAVAILABLE'); + expect(res.payload.error.message).toMatch(/planUpgrade/); + expect(door.register).not.toHaveBeenCalled(); + expect(ledgerFiles()).toEqual(before); + }); +}); + +describe('a dryRun this door cannot honour is refused, never run as an install', () => { + it('a dryRun that is not a boolean', async () => { + const door = await mount(); + const res = await door.install(makeC({ manifest: V1, dryRun: 'yes' })); + expect(res.status).toBe(400); + expect(res.payload.error).toMatchObject({ code: 'INVALID_REQUEST' }); + expect(res.payload.error.message).toMatch(/`dryRun` must be true or false/); + expect(door.register).not.toHaveBeenCalled(); + expect(ledgerFiles()).toEqual({}); + }); + + it('a dry run of an unmanaged install', async () => { + const door = await mount(); + const res = await door.install(makeC({ manifest: V1, dryRun: true, installMode: 'unmanaged' })); + expect(res.status).toBe(400); + expect(res.payload.error.message).toMatch(/an `unmanaged` install is a copy that follows no upgrade/); + expect(door.register).not.toHaveBeenCalled(); + expect(ledgerFiles()).toEqual({}); + }); +}); From 3dba41198913272f30da1e8c557a269aad900e2f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:21:37 +0000 Subject: [PATCH 6/7] feat(cli): os package install --dry-run prints the upgrade plan `--dry-run` sends `dryRun: true` and prints the runtime's plan, each finding with the environment's reading beside it. A runtime that ignored the key and installed is printed as a failure, exit 1. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- .changeset/30884-package-upgrade-plan.md | 19 +++ packages/cli/src/commands/package/install.ts | 122 +++++++++++++++ .../cli/test/package-install-dry-run.test.ts | 143 ++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 .changeset/30884-package-upgrade-plan.md create mode 100644 packages/cli/test/package-install-dry-run.test.ts diff --git a/.changeset/30884-package-upgrade-plan.md b/.changeset/30884-package-upgrade-plan.md new file mode 100644 index 00000000000..923a790baf5 --- /dev/null +++ b/.changeset/30884-package-upgrade-plan.md @@ -0,0 +1,19 @@ +--- +'@objectstack/metadata-core': patch +'@objectstack/objectql': patch +'@objectstack/cloud-connection': patch +'@objectstack/cli': patch +--- + +feat: the package upgrade plan — one comparison of an installed package version with the incoming one, a dry run of the install door, and `os package install --dry-run` + +Clause-②: yes (widening) — new exports from `@objectstack/metadata-core` (`comparePackageVersions` and its types, `typeChangeCategory`, `isEmptyPackageUpgradePlan`, `upgradePlanSeedDatasets`, `seedModeOverwritesMatchedRows`), a new `planUpgrade` method on objectql's `manifest` service, a new `dryRun` key on the `POST /api/v1/marketplace/install-local` body, a new `--dry-run` flag on `os package install`, and new `os diff` entries (declared and suspected renames, seed datasets). Nothing a request did before answers differently. +Metadata-protocol: none + +**One comparison.** `comparePackageVersions(installed, incoming)` (`@objectstack/metadata-core`) lists what an upgrade from one version of a package to the next changes: objects and fields added and removed; type changes, each with the drift category `schema-drift.ts` uses (`safe`, `needs_confirm`, `destructive`); renames a field declares with `renamedFrom`; suspected renames, where one object loses a stored field and gains one with nothing linking them; and seed datasets added, removed and changed, each with its mode. A declared rename whose type changes is also listed as a type change. It is pure and decides nothing. + +**What the environment knows.** The `manifest` service's `planUpgrade(installed, incoming)` (`@objectstack/objectql`) answers the comparison with two readings of the installed data: the rows holding a value in each column the upgrade removes, renames or retypes (and the rows of each object it removes), and, for each `upsert` or `update` dataset both versions declare, the rows a customer edited that the seed replay would write over. A reading that fails is `null`, never `0`. + +**A dry run.** `POST /api/v1/marketplace/install-local` with `dryRun: true` answers `200` with the plan of the installed version (its ledger entry) against the one sent, and installs and writes nothing. A dry run of an `unmanaged` install or of a customization pair is refused `400 INVALID_REQUEST`, as is a `dryRun` that is not a boolean. `os package install … --dry-run` sends it and prints the plan. A runtime that predates the key ignores it and installs; the command then says so and exits 1. + +**`os diff`** reads its object and field changes from the same comparison, so the CLI and an install cannot disagree. Its output for every shape it reported before is unchanged. New: a field renamed with `renamedFrom` prints as one rename instead of a removal and an addition (`renamedFrom` is new in this release), a suspected rename prints beside its removal and addition, and seed datasets print under `data`. diff --git a/packages/cli/src/commands/package/install.ts b/packages/cli/src/commands/package/install.ts index 7dd59d3734f..8d62991ed2c 100644 --- a/packages/cli/src/commands/package/install.ts +++ b/packages/cli/src/commands/package/install.ts @@ -41,6 +41,14 @@ * the pair install, and registered it as read-only code) is printed as the * failure it is, with the way back. * + * `--dry-run` (epic cloud#3652 Q3, #30884) installs nothing: the runtime + * answers the package upgrade plan — the version it has installed against + * this one: objects, fields, type changes, declared (`renamedFrom`) and + * suspected renames, seed datasets — with the rows each touched column holds + * and the rows a customer edited that the seed replay would overwrite, and + * the command prints it. A runtime that predates the dry run ignores the key + * and installs; the command reads the answer, and says so when that happened. + * * This complements `os package publish` (which uploads to the CLOUD): the * pairing is publish-to-cloud / install-into-runtime. The target runtime * authenticates the call itself — pass a token it issued (-t/--token, or @@ -76,6 +84,86 @@ import { printHeader, printKV, printSuccess, printWarning, printError, printStep * withheld, or a runtime that predates the key) prints nothing, and nothing * is guessed from the artifact the command sent. */ +/** One row count, or the honest "could not be read". */ +function rowsText(rows: unknown, noun: string): string { + if (rows === null) return 'its rows could not be read'; + if (typeof rows !== 'number') return ''; + return `${rows} row(s) ${noun}`; +} + +/** + * [epic cloud#3652 Q3, #30884] The lines a dry run's answer prints: the + * package upgrade plan (`data.plan`, `comparePackageVersions`) with the + * environment's readings beside each finding (`data.environment`). Printed + * from the answer only; a list the answer does not carry prints nothing. + */ +export function renderUpgradePlan(data: any): string[] { + const plan = data?.plan ?? {}; + const env = data?.environment ?? {}; + const lines: string[] = []; + const columnRows = new Map(); + for (const c of Array.isArray(env.columns) ? env.columns : []) columnRows.set(`${c.object}.${c.field}`, c.rows); + const objectRows = new Map(); + for (const o of Array.isArray(env.removedObjects) ? env.removedObjects : []) objectRows.set(o.object, o.rows); + const list = (value: unknown): any[] => (Array.isArray(value) ? value : []); + const held = (object: string, field: string) => { + const text = rowsText(columnRows.get(`${object}.${field}`), 'hold a value'); + return text ? ` — ${text}` : ''; + }; + + const section = (title: string, items: string[]) => { + if (items.length === 0) return; + lines.push(` ${title}`); + for (const item of items) lines.push(` ${item}`); + }; + section('Objects', [ + ...list(plan.objects?.added).map((name: string) => `+ ${name}`), + ...list(plan.objects?.removed).map((name: string) => { + const text = rowsText(objectRows.get(name), 'in it'); + return `- ${name}${text ? ` — ${text}; the table is not dropped` : ''}`; + }), + ]); + section('Fields', [ + ...list(plan.fields?.added).map((f: any) => `+ ${f.object}.${f.field}${f.type ? ` (${f.type})` : ''}${f.required ? ' required' : ''}`), + ...list(plan.fields?.removed).map((f: any) => `- ${f.object}.${f.field}${f.type ? ` (${f.type})` : ''}${held(f.object, f.field)}; the column is not dropped`), + ]); + section('Renames declared by renamedFrom', list(plan.declaredRenames).map((r: any) => + `${r.object}.${r.from} → ${r.to}${held(r.object, r.from)}`)); + section('Suspected renames (no renamedFrom links them)', list(plan.suspectedRenames).map((r: any) => + `${r.object}.${r.field} → ${list(r.candidates).map((c: any) => c.field).join(' or ')}?${held(r.object, r.field)}; ` + + 'the new key would install as a new empty column. Declare renamedFrom on the new field and rebuild if it is a rename')); + section('Type changes', list(plan.typeChanges).map((t: any) => + `${t.object}.${t.field}: ${t.from} → ${t.to} (${t.category})${t.renamedFrom ? `, renamed from ${t.renamedFrom}` : ''}` + + `${held(t.object, t.renamedFrom ?? t.field)}`)); + + const overwrites = new Map(); + for (const o of list(env.seeds)) overwrites.set(o.key, o); + const seedLine = (d: any, sign: string) => { + const o = overwrites.get(d.key); + let line = `${sign} ${d.key} (mode ${d.fromMode && d.fromMode !== d.mode ? `${d.fromMode} → ${d.mode}` : d.mode}, ${d.records} record(s))`; + if (Array.isArray(d.changed)) line += `: ${d.changed.join(', ')} changed`; + if (o) { + line += o.editedRows === null + ? ' — rows a customer edited could not be read' + : ` — ${o.editedRows} row(s) a customer edited would be overwritten${o.editedKeys?.length ? ` (${o.editedKeys.join(', ')})` : ''}`; + } + return line; + }; + const changedKeys = new Set(list(plan.seeds?.changed).map((d: any) => d.key)); + section('Seed datasets', [ + ...list(plan.seeds?.added).map((d: any) => seedLine(d, '+')), + ...list(plan.seeds?.removed).map((d: any) => `- ${d.key} (mode ${d.mode}); its rows are not removed`), + ...list(plan.seeds?.changed).map((d: any) => seedLine(d, '~')), + // An unchanged overwriting dataset still writes over the rows a customer edited. + ...list(env.seeds).filter((o: any) => !changedKeys.has(o.key) && o.editedRows !== 0) + .map((o: any) => `= ${o.key} (mode ${o.mode}, unchanged)${o.editedRows === null + ? ' — rows a customer edited could not be read' + : ` — ${o.editedRows} row(s) a customer edited would be overwritten${o.editedKeys?.length ? ` (${o.editedKeys.join(', ')})` : ''}`}`), + ]); + if (lines.length === 0) lines.push(' No change to objects, fields or seed datasets.'); + return lines; +} + function printWithheldMaps(withheldMaps: unknown): void { const maps: Array<{ type?: unknown; name?: unknown; path?: unknown }> = Array.isArray(withheldMaps) ? withheldMaps : []; @@ -100,6 +188,7 @@ export default class PackageInstall extends Command { '$ os package install ./dist/objectstack.json --install-mode unmanaged # copy into the environment as its own metadata', '$ OS_RUNTIME_EMAIL=admin@local.test OS_RUNTIME_PASSWORD=… os package install com.acme.crm', '$ os package install ./dist/objectstack.json --url https://app.example.com --token osk_…', + '$ os package install ./dist/objectstack.json --dry-run # print the upgrade plan, install nothing', ]; static override args = { @@ -176,6 +265,14 @@ export default class PackageInstall extends Command { "Confirm the access findings of this upgrade with the finding-set token the runtime's refusal printed " + '(ACCESS_UPGRADE_CONFIRMATION_REQUIRED). It confirms exactly those findings; a changed set is refused again.', }), + // [epic cloud#3652 Q3, #30884] The install-local body's `dryRun`. + 'dry-run': Flags.boolean({ + description: + 'Install nothing: print the upgrade plan of the installed version against this one (objects, fields, type ' + + 'changes, declared and suspected renames, seed datasets), with the rows each touched column holds and the ' + + 'rows a customer edited that the seed replay would overwrite', + default: false, + }), }; async run(): Promise { @@ -274,6 +371,7 @@ export default class PackageInstall extends Command { if (flags['confirm-access-upgrade'] !== undefined) body.confirmAccessUpgrade = flags['confirm-access-upgrade']; const installMode = flags['install-mode'] as InstallMode | undefined; if (installMode !== undefined) body.installMode = installMode; + if (flags['dry-run']) body.dryRun = true; const res = await this.request(`${runtime}/api/v1/marketplace/install-local`, { method: 'POST', headers: { @@ -344,6 +442,30 @@ export default class PackageInstall extends Command { const data = res.body?.data ?? res.body ?? {}; + // [#30884] The dry run's answer: the plan, and nothing installed. + if (data.dryRun === true) { + console.log(''); + printSuccess('Dry run — nothing was installed or written'); + printKV(' Package', String(data.manifestId ?? label)); + printKV(' Version', `${String(data.version ?? flags.version)}${typeof data.installedVersion === 'string' ? ` (installed: ${data.installedVersion})` : ' (not installed)'}`); + printKV(' Runtime', runtime); + console.log(''); + for (const line of renderUpgradePlan(data)) console.log(line); + console.log(''); + return; + } + // Asked for a dry run, answered without it: the runtime predates the + // key, read no `dryRun`, and INSTALLED. Said out loud, never printed as + // a plan — the opposite of what was asked happened. + if (flags['dry-run']) { + printError( + `The runtime at ${runtime} does not support a dry run: it ignored \`--dry-run\` and INSTALLED ${label}. ` + + 'Upgrade the runtime to preview an upgrade before it is installed.', + ); + this.exit(1); + return; + } + // [ADR-0146 D3, #30106] A base's paired customization package answers // its own shape too: the writable pair its items were written into, // never a kernel registration or a cached manifest — so, as for an diff --git a/packages/cli/test/package-install-dry-run.test.ts b/packages/cli/test/package-install-dry-run.test.ts new file mode 100644 index 00000000000..0800e8b6491 --- /dev/null +++ b/packages/cli/test/package-install-dry-run.test.ts @@ -0,0 +1,143 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [epic cloud#3652 Q3, #30884] `os package install --dry-run` — the + * install-local body's `dryRun`, and how the command prints the runtime's + * upgrade plan. + * + * Pinned: + * - the flag reaches the body as `dryRun: true`; without it the body carries + * no such key; + * - the answer prints as a dry run (nothing installed), each finding with + * the environment's reading beside it: rows in a touched column, rows a + * customer edited that the seed replay would overwrite, a reading that + * failed as "could not be read"; + * - a runtime that ignored the key and installed is printed as the failure + * it is, exit 1. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest'; +import PackageInstall, { renderUpgradePlan } from '../src/commands/package/install.js'; + +type Answer = { ok: boolean; status: number; body: unknown }; + +function stubRuntime(answer: Answer): { sent: Array> } { + const sent: Array> = []; + vi.stubGlobal('fetch', vi.fn(async (_url: string, init?: { body?: string }) => { + if (init?.body) sent.push(JSON.parse(init.body)); + return { + ok: answer.ok, + status: answer.status, + statusText: answer.ok ? 'OK' : 'Refused', + headers: { get: () => null }, + json: async () => answer.body, + } as any; + })); + return { sent }; +} + +async function runInstall(args: string[]): Promise<{ out: string; exit: number | undefined }> { + const lines: string[] = []; + const capture = (...a: unknown[]) => { lines.push(a.map(String).join(' ')); }; + vi.spyOn(console, 'log').mockImplementation(capture); + vi.spyOn(console, 'error').mockImplementation(capture); + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { lines.push(String(chunk)); return true; }) as any); + let exit: number | undefined; + try { + await PackageInstall.run(['com.acme.crm', '--url', 'http://runtime.test', ...args]); + } catch (err: any) { + exit = err?.oclif?.exit ?? err?.code ?? 1; + } + return { out: lines.join('\n'), exit }; +} + +const PLAN_ANSWER = { + dryRun: true, + manifestId: 'com.acme.crm', + version: '2.0.0', + installedVersion: '1.0.0', + plan: { + objects: { added: [], removed: ['acme_old'] }, + fields: { + added: [{ object: 'acme_lead', field: 'account_name', type: 'text', required: false, stored: true }], + removed: [{ object: 'acme_lead', field: 'company', type: 'text', required: false, stored: true }], + }, + typeChanges: [{ object: 'acme_deal', field: 'amount_value', from: 'text', to: 'number', category: 'needs_confirm', renamedFrom: 'amount' }], + declaredRenames: [{ object: 'acme_deal', from: 'amount', to: 'amount_value', fromType: 'text', toType: 'number' }], + suspectedRenames: [{ object: 'acme_lead', field: 'company', type: 'text', candidates: [{ field: 'account_name', type: 'text' }] }], + seeds: { + added: [], + removed: [], + changed: [{ key: 'acme_stage', object: 'acme_stage', externalId: 'name', mode: 'upsert', records: 3, fromMode: 'upsert', fromRecords: 3, changed: ['records'] }], + }, + }, + environment: { + columns: [ + { object: 'acme_lead', field: 'company', reasons: ['removed', 'suspected_rename'], rows: 42 }, + { object: 'acme_deal', field: 'amount', reasons: ['renamed', 'type_change'], rows: null }, + ], + removedObjects: [{ object: 'acme_old', rows: 7 }], + seeds: [ + { key: 'acme_stage', object: 'acme_stage', mode: 'upsert', changed: true, matchedRows: 3, editedRows: 1, editedKeys: ['won'] }, + { key: 'acme_tier', object: 'acme_tier', mode: 'upsert', changed: false, matchedRows: 2, editedRows: 2, editedKeys: ['gold', 'silver'] }, + ], + }, +}; + +describe('os package install --dry-run (#30884)', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('sends `dryRun: true`, and no such key without the flag', async () => { + const { sent } = stubRuntime({ ok: true, status: 200, body: { success: true, data: PLAN_ANSWER } }); + await runInstall(['--dry-run']); + expect(sent[0]).toMatchObject({ packageId: 'com.acme.crm', dryRun: true }); + + vi.unstubAllGlobals(); + const second = stubRuntime({ ok: true, status: 200, body: { success: true, data: { manifestId: 'com.acme.crm', version: '2.0.0', hotLoaded: true } } }); + await runInstall([]); + expect(second.sent[0]).not.toHaveProperty('dryRun'); + }); + + it('prints the plan with each reading beside its finding, and says nothing was installed', async () => { + stubRuntime({ ok: true, status: 200, body: { success: true, data: PLAN_ANSWER } }); + const { out, exit } = await runInstall(['--dry-run']); + expect(exit).toBeUndefined(); + expect(out).toContain('Dry run — nothing was installed or written'); + expect(out).toContain('2.0.0 (installed: 1.0.0)'); + expect(out).not.toContain('Package installed'); + }); + + it('the plan lines', () => { + expect(renderUpgradePlan(PLAN_ANSWER)).toEqual([ + ' Objects', + ' - acme_old — 7 row(s) in it; the table is not dropped', + ' Fields', + ' + acme_lead.account_name (text)', + ' - acme_lead.company (text) — 42 row(s) hold a value; the column is not dropped', + ' Renames declared by renamedFrom', + ' acme_deal.amount → amount_value — its rows could not be read', + ' Suspected renames (no renamedFrom links them)', + ' acme_lead.company → account_name? — 42 row(s) hold a value; the new key would install as a new empty column. ' + + 'Declare renamedFrom on the new field and rebuild if it is a rename', + ' Type changes', + ' acme_deal.amount_value: text → number (needs_confirm), renamed from amount — its rows could not be read', + ' Seed datasets', + ' ~ acme_stage (mode upsert, 3 record(s)): records changed — 1 row(s) a customer edited would be overwritten (won)', + ' = acme_tier (mode upsert, unchanged) — 2 row(s) a customer edited would be overwritten (gold, silver)', + ]); + }); + + it('an empty plan says so', () => { + expect(renderUpgradePlan({ plan: {}, environment: {} })).toEqual([' No change to objects, fields or seed datasets.']); + }); + + it('a runtime that ignored --dry-run and installed is a failure, exit 1', async () => { + stubRuntime({ ok: true, status: 200, body: { success: true, data: { manifestId: 'com.acme.crm', version: '2.0.0', hotLoaded: true } } }); + const { out, exit } = await runInstall(['--dry-run']); + expect(exit).toBe(1); + expect(out).toContain('does not support a dry run: it ignored `--dry-run` and INSTALLED com.acme.crm@latest'); + }); +}); From 3100b841e32027af5f3d910cba6c6b70c4ca1b19 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 14:38:28 +0000 Subject: [PATCH 7/7] test(objectql): the failed-reading case stubs the engine read, not a driver double `check:objectql-double-limit` drives every `find` double it finds with its own rows; the in-memory driver's per-object failure switch threw under that probe and was left unjudged. The failure case now rejects `engine.find` itself, and the driver double is plain. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_016WV4cyPqWLpc3AxaZrs98n --- .../src/package-upgrade-environment.test.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/objectql/src/package-upgrade-environment.test.ts b/packages/objectql/src/package-upgrade-environment.test.ts index 21da73adeab..a96ab7886da 100644 --- a/packages/objectql/src/package-upgrade-environment.test.ts +++ b/packages/objectql/src/package-upgrade-environment.test.ts @@ -16,7 +16,7 @@ import { planPackageUpgrade } from './package-upgrade-environment.js'; type Row = Record; /** In-memory driver: equality, `$in` and `$ne` WHERE; `find` honours `limit`. */ -function makeDriver(rowsByObject: Record, failing: ReadonlySet = new Set()) { +function makeDriver(rowsByObject: Record) { const matches = (row: Row, where: any): boolean => { if (!where || typeof where !== 'object') return true; for (const [k, v] of Object.entries(where)) { @@ -38,10 +38,7 @@ function makeDriver(rowsByObject: Record, failing: ReadonlySet { - if (failing.has(object)) throw new Error(`connection lost reading ${object}`); - return (rowsByObject[object] ?? []).map((row) => ({ ...row })); - }; + const rowsOf = (object: string) => (rowsByObject[object] ?? []).map((row) => ({ ...row })); return { name: 'memory', version: '0.0.0', supports: {} as any, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, @@ -61,9 +58,9 @@ function makeDriver(rowsByObject: Record, failing: ReadonlySet }>, rows: Record, failing?: Set) { +async function makeEngine(objects: Array<{ name: string; fields: Record }>, rows: Record) { const engine = new ObjectQL(); - engine.registerDriver(makeDriver(rows, failing) as any, true); + engine.registerDriver(makeDriver(rows) as any, true); await engine.init(); for (const object of objects) (engine.registry as any).registerObject(object); return engine; @@ -130,7 +127,8 @@ describe('rows a customer edited that the seed replay overwrites', () => { it('a reading that fails is null, never zero, and is told', async () => { const warn = vi.fn(); const body = { objects: [STAGE], data: [{ object: 'acme_stage', mode: 'upsert', records: stages({ won: 'Won' }) }] }; - const engine = await makeEngine([STAGE], {}, new Set(['acme_stage'])); + const engine = await makeEngine([STAGE], {}); + vi.spyOn(engine, 'find').mockRejectedValue(new Error('connection lost reading acme_stage')); const { environment } = await planPackageUpgrade(engine, body, structuredClone(body), { warn }); expect(environment.seeds).toEqual([expect.objectContaining({ matchedRows: null, editedRows: null })]); expect(warn).toHaveBeenCalledWith(expect.stringContaining('answered null, never zero'), expect.objectContaining({ object: 'acme_stage' }));