feat: the package upgrade plan — compare the installed and incoming versions, an install dry run, os package install --dry-run

.changeset/30884-package-upgrade-plan.md

@@ -0,0 +1,19 @@
+---
+'@objectstack/metadata-core': patch
+'@objectstack/objectql': patch
+'@objectstack/cloud-connection': patch
+'@objectstack/cli': patch
+---
+
+feat: the package upgrade plan — one comparison of an installed package version with the incoming one, a dry run of the install door, and `os package install --dry-run`
+
+Clause-②: yes (widening) — new exports from `@objectstack/metadata-core` (`comparePackageVersions` and its types, `typeChangeCategory`, `isEmptyPackageUpgradePlan`, `upgradePlanSeedDatasets`, `seedModeOverwritesMatchedRows`), a new `planUpgrade` method on objectql's `manifest` service, a new `dryRun` key on the `POST /api/v1/marketplace/install-local` body, a new `--dry-run` flag on `os package install`, and new `os diff` entries (declared and suspected renames, seed datasets). Nothing a request did before answers differently.
+Metadata-protocol: none
+
+**One comparison.** `comparePackageVersions(installed, incoming)` (`@objectstack/metadata-core`) lists what an upgrade from one version of a package to the next changes: objects and fields added and removed; type changes, each with the drift category `schema-drift.ts` uses (`safe`, `needs_confirm`, `destructive`); renames a field declares with `renamedFrom`; suspected renames, where one object loses a stored field and gains one with nothing linking them; and seed datasets added, removed and changed, each with its mode. A declared rename whose type changes is also listed as a type change. It is pure and decides nothing.
+
+**What the environment knows.** The `manifest` service's `planUpgrade(installed, incoming)` (`@objectstack/objectql`) answers the comparison with two readings of the installed data: the rows holding a value in each column the upgrade removes, renames or retypes (and the rows of each object it removes), and, for each `upsert` or `update` dataset both versions declare, the rows a customer edited that the seed replay would write over. A reading that fails is `null`, never `0`.
+
+**A dry run.** `POST /api/v1/marketplace/install-local` with `dryRun: true` answers `200` with the plan of the installed version (its ledger entry) against the one sent, and installs and writes nothing. A dry run of an `unmanaged` install or of a customization pair is refused `400 INVALID_REQUEST`, as is a `dryRun` that is not a boolean. `os package install … --dry-run` sends it and prints the plan. A runtime that predates the key ignores it and installs; the command then says so and exits 1.
+
+**`os diff`** reads its object and field changes from the same comparison, so the CLI and an install cannot disagree. Its output for every shape it reported before is unchanged. New: a field renamed with `renamedFrom` prints as one rename instead of a removal and an addition (`renamedFrom` is new in this release), a suspected rename prints beside its removal and addition, and seed datasets print under `data`.

packages/cli/src/commands/diff.ts

@@ -2,6 +2,7 @@
 
 import { Args, Command, Flags } from '@oclif/core';
 import chalk from 'chalk';
+import { comparePackageVersions, seedModeOverwritesMatchedRows } from '@objectstack/metadata-core';
 import { loadConfig } from '../utils/config.js';
 import { authoringRuleUnionStack } from '../utils/stack-collections.js';
 import {
@@ -20,7 +21,7 @@ import {
 
 // ─── Types ──────────────────────────────────────────────────────────
 
-interface DiffEntry {
+export interface DiffEntry {
   type: 'added' | 'removed' | 'modified';
   category: string;
   name: string;
@@ -39,24 +40,10 @@ function getNames(items: any[] | undefined): Map<string, any> {
   return map;
 }
 
-function getFieldNames(obj: any): string[] {
-  if (!obj?.fields || typeof obj.fields !== 'object') return [];
-  return Object.keys(obj.fields);
-}
-
-function getFieldType(obj: any, fieldName: string): string | undefined {
-  return obj?.fields?.[fieldName]?.type;
-}
-
-function isFieldRequired(obj: any, fieldName: string): boolean {
-  return obj?.fields?.[fieldName]?.required === true;
-}
-
 function diffNamedArrays(
   beforeItems: any[] | undefined,
   afterItems: any[] | undefined,
   category: string,
-  detectFieldChanges: boolean,
 ): DiffEntry[] {
   const entries: DiffEntry[] = [];
   const beforeMap = getNames(beforeItems);
@@ -86,76 +73,136 @@ function diffNamedArrays(
     }
   }
 
-  // Modified items — field-level diff for objects
-  if (detectFieldChanges) {
-    for (const [name, beforeObj] of beforeMap) {
-      const afterObj = afterMap.get(name);
-      if (!afterObj) continue;
-
-      const beforeFields = getFieldNames(beforeObj);
-      const afterFields = getFieldNames(afterObj);
-      const beforeSet = new Set(beforeFields);
-      const afterSet = new Set(afterFields);
-
-      // Removed fields
-      for (const f of beforeFields) {
-        if (!afterSet.has(f)) {
-          entries.push({
-            type: 'removed',
-            category: `${category}.${name}.fields`,
-            name: f,
-            breaking: true,
-            detail: 'field removed',
-          });
-        }
-      }
+  return entries;
+}
 
-      // Added fields
-      for (const f of afterFields) {
-        if (!beforeSet.has(f)) {
-          const breaking = isFieldRequired(afterObj, f);
-          entries.push({
-            type: 'added',
-            category: `${category}.${name}.fields`,
-            name: f,
-            breaking,
-            detail: breaking ? 'required field added' : 'optional field added',
-          });
-        }
-      }
+function seedModeNote(mode: string): string {
+  if (seedModeOverwritesMatchedRows(mode)) return '; a replay writes its columns over the rows it matches';
+  if (mode === 'replace') return '; a replay deletes every row of the object and inserts these';
+  return '';
+}
 
-      // Type changes
-      for (const f of beforeFields) {
-        if (!afterSet.has(f)) continue;
-        const oldType = getFieldType(beforeObj, f);
-        const newType = getFieldType(afterObj, f);
-        if (oldType && newType && oldType !== newType) {
-          entries.push({
-            type: 'modified',
-            category: `${category}.${name}.fields`,
-            name: f,
-            breaking: true,
-            detail: `type changed: ${oldType} → ${newType}`,
-          });
-        }
-      }
+/**
+ * Objects, fields and seed datasets, read from the package upgrade plan
+ * (`comparePackageVersions`, `@objectstack/metadata-core`) — the comparison
+ * the install door's dry run and the `manifest` service's `planUpgrade` read,
+ * so this command and an install cannot disagree about one pair of versions.
+ *
+ * Printed per object as before: removed fields, added fields, then (new with
+ * the plan) declared renames, type changes, then suspected renames, then the
+ * object's label change. A suspected rename's two keys stay listed as removed
+ * and added, as the plan lists them: nothing declares otherwise.
+ */
+function diffObjectsAndSeeds(beforeConfig: any, afterConfig: any): DiffEntry[] {
+  const plan = comparePackageVersions(beforeConfig, afterConfig);
+  const entries: DiffEntry[] = [];
 
-      // Label / ownership changes on the object itself
-      if (beforeObj.label !== afterObj.label) {
-        entries.push({
-          type: 'modified',
-          category,
-          name,
-          breaking: false,
-          detail: `label changed: "${beforeObj.label ?? '(none)'}" → "${afterObj.label ?? '(none)'}"`,
-        });
-      }
+  for (const name of plan.objects.removed) entries.push({ type: 'removed', category: 'objects', name, breaking: true });
+  for (const name of plan.objects.added) entries.push({ type: 'added', category: 'objects', name, breaking: false });
+
+  const afterObjects = getNames(afterConfig.objects);
+  for (const [name, beforeObj] of getNames(beforeConfig.objects)) {
+    const afterObj = afterObjects.get(name);
+    if (!afterObj) continue;
+    const category = `objects.${name}.fields`;
+    const mine = <T extends { object: string }>(list: T[]): T[] => list.filter((item) => item.object === name);
+
+    for (const f of mine(plan.fields.removed)) {
+      entries.push({ type: 'removed', category, name: f.field, breaking: true, detail: 'field removed' });
+    }
+    for (const f of mine(plan.fields.added)) {
+      entries.push({
+        type: 'added',
+        category,
+        name: f.field,
+        breaking: f.required,
+        detail: f.required ? 'required field added' : 'optional field added',
+      });
+    }
+    for (const r of mine(plan.declaredRenames)) {
+      entries.push({ type: 'modified', category, name: r.to, breaking: true, detail: `renamed from ${r.from}, declared by renamedFrom` });
+    }
+    for (const t of mine(plan.typeChanges)) {
+      entries.push({ type: 'modified', category, name: t.field, breaking: true, detail: `type changed: ${t.from} → ${t.to}` });
+    }
+    for (const s of mine(plan.suspectedRenames)) {
+      entries.push({
+        type: 'modified',
+        category,
+        name: s.field,
+        breaking: true,
+        detail: `suspected rename to ${s.candidates.map((c) => c.field).join(' or ')}: no renamedFrom declares it, `
+          + 'so the new key installs as a new empty column',
+      });
+    }
+
+    // Label / ownership changes on the object itself
+    if (beforeObj.label !== afterObj.label) {
+      entries.push({
+        type: 'modified',
+        category: 'objects',
+        name,
+        breaking: false,
+        detail: `label changed: "${beforeObj.label ?? '(none)'}" → "${afterObj.label ?? '(none)'}"`,
+      });
     }
   }
 
+  for (const d of plan.seeds.removed) {
+    entries.push({ type: 'removed', category: 'data', name: d.key, breaking: false, detail: `seed dataset removed, mode ${d.mode}` });
+  }
+  for (const d of plan.seeds.added) {
+    entries.push({
+      type: 'added',
+      category: 'data',
+      name: d.key,
+      breaking: false,
+      detail: `seed dataset added: ${d.records} record(s); mode ${d.mode}${seedModeNote(d.mode)}`,
+    });
+  }
+  for (const d of plan.seeds.changed) {
+    const mode = d.fromMode === d.mode ? d.mode : `${d.fromMode} → ${d.mode}`;
+    entries.push({
+      type: 'modified',
+      category: 'data',
+      name: d.key,
+      breaking: false,
+      detail: `seed dataset changed: ${d.changed.join(', ')}; mode ${mode}${seedModeNote(d.mode)}`,
+    });
+  }
+
   return entries;
 }
 
+/**
+ * Every change between two folded stacks, in the order the command prints
+ * them: objects with their field-level changes first, then each name-level
+ * category, then seed datasets.
+ */
+export function diffStacks(beforeConfig: any, afterConfig: any): DiffEntry[] {
+  const plan = diffObjectsAndSeeds(beforeConfig, afterConfig);
+  const allDiffs: DiffEntry[] = plan.filter((entry) => entry.category !== 'data');
+
+  // Views, Flows, Agents, Apps (name-level diff)
+  const simpleCats: Array<{ key: string; label: string }> = [
+    { key: 'views', label: 'views' },
+    { key: 'flows', label: 'flows' },
+    { key: 'agents', label: 'agents' },
+    { key: 'apps', label: 'apps' },
+    { key: 'dashboards', label: 'dashboards' },
+    { key: 'actions', label: 'actions' },
+    { key: 'workflows', label: 'workflows' },
+    { key: 'apis', label: 'apis' },
+    { key: 'positions', label: 'positions' },
+  ];
+
+  for (const cat of simpleCats) {
+    allDiffs.push(...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label));
+  }
+  allDiffs.push(...plan.filter((entry) => entry.category === 'data'));
+  return allDiffs;
+}
+
 // ─── Command ────────────────────────────────────────────────────────
 
 export default class Diff extends Command {
@@ -229,29 +276,7 @@ export default class Diff extends Command {
       }
 
       // ── Diff all categories ──
-      const allDiffs: DiffEntry[] = [];
-
-      // Objects (with field-level diff)
-      allDiffs.push(...diffNamedArrays(beforeConfig.objects, afterConfig.objects, 'objects', true));
-
-      // Views, Flows, Agents, Apps (name-level diff)
-      const simpleCats: Array<{ key: string; label: string }> = [
-        { key: 'views', label: 'views' },
-        { key: 'flows', label: 'flows' },
-        { key: 'agents', label: 'agents' },
-        { key: 'apps', label: 'apps' },
-        { key: 'dashboards', label: 'dashboards' },
-        { key: 'actions', label: 'actions' },
-        { key: 'workflows', label: 'workflows' },
-        { key: 'apis', label: 'apis' },
-        { key: 'positions', label: 'positions' },
-      ];
-
-      for (const cat of simpleCats) {
-        allDiffs.push(
-          ...diffNamedArrays(beforeConfig[cat.key], afterConfig[cat.key], cat.label, false),
-        );
-      }
+      const allDiffs = diffStacks(beforeConfig, afterConfig);
 
       // ── Filter ──
       const diffs = flags['breaking-only']