From 3a7891b74d3e9ed12a1fd8ee0d16c6715428d2d6 Mon Sep 17 00:00:00 2001 From: Abe Wheeler Date: Sun, 25 Oct 2026 15:36:35 -0500 Subject: [PATCH] Preserve retryable historical validation on Git deadlines --- package-lock.json | 8 +- package.json | 2 +- packages/create-filegrc/package.json | 2 +- packages/create-filegrc/src/index.js | 4 +- packages/create-filegrc/template/package.json | 2 +- packages/filegrc/package.json | 2 +- packages/filegrc/src/git.js | 11 +++ .../filegrc/src/nonmaterial-amendments.js | 6 +- .../filegrc/src/retention-applications.js | 4 +- .../test/nonmaterial-amendments.test.js | 23 ++++++ .../filegrc/test/operating-baseline.test.js | 9 +++ .../test/retention-applications.test.js | 73 ++++++++++++++++++- 12 files changed, 129 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index add54cb..6580a3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "filegrc-monorepo", - "version": "0.20.6", + "version": "0.20.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "filegrc-monorepo", - "version": "0.20.6", + "version": "0.20.7", "workspaces": [ "packages/filegrc", "packages/create-filegrc" @@ -24,7 +24,7 @@ "link": true }, "packages/create-filegrc": { - "version": "0.20.6", + "version": "0.20.7", "license": "MIT", "bin": { "create-filegrc": "bin/create-filegrc.js" @@ -34,7 +34,7 @@ } }, "packages/filegrc": { - "version": "0.20.6", + "version": "0.20.7", "license": "MIT", "bin": { "filegrc": "bin/filegrc.js" diff --git a/package.json b/package.json index a1747a6..002c2e8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "filegrc-monorepo", - "version": "0.20.6", + "version": "0.20.7", "private": true, "workspaces": [ "packages/filegrc", diff --git a/packages/create-filegrc/package.json b/packages/create-filegrc/package.json index d5f85a8..d3110dd 100644 --- a/packages/create-filegrc/package.json +++ b/packages/create-filegrc/package.json @@ -1,6 +1,6 @@ { "name": "create-filegrc", - "version": "0.20.6", + "version": "0.20.7", "description": "Create a filegrc workspace for a SOC 2 program", "license": "MIT", "repository": { diff --git a/packages/create-filegrc/src/index.js b/packages/create-filegrc/src/index.js index 14a0ff3..afda92c 100644 --- a/packages/create-filegrc/src/index.js +++ b/packages/create-filegrc/src/index.js @@ -524,13 +524,13 @@ async function runCombinedSetup(target, input) { async function writeMinimalLockfile(target, name, versionRange) { const lock = { name, - version: "0.20.6", + version: "0.20.7", lockfileVersion: 3, requires: true, packages: { "": { name, - version: "0.20.6", + version: "0.20.7", dependencies: { filegrc: versionRange } } } diff --git a/packages/create-filegrc/template/package.json b/packages/create-filegrc/template/package.json index e214ebf..d58db38 100644 --- a/packages/create-filegrc/template/package.json +++ b/packages/create-filegrc/template/package.json @@ -1,6 +1,6 @@ { "name": "{{project_name}}", - "version": "0.20.6", + "version": "0.20.7", "private": true, "description": "filegrc workspace for a SOC 2 program", "type": "module", diff --git a/packages/filegrc/package.json b/packages/filegrc/package.json index cdb303f..3f1a3d4 100644 --- a/packages/filegrc/package.json +++ b/packages/filegrc/package.json @@ -1,6 +1,6 @@ { "name": "filegrc", - "version": "0.20.6", + "version": "0.20.7", "description": "Zero-dependency Git-native GRC engine", "license": "MIT", "repository": { diff --git a/packages/filegrc/src/git.js b/packages/filegrc/src/git.js index 4711f5b..0029b75 100644 --- a/packages/filegrc/src/git.js +++ b/packages/filegrc/src/git.js @@ -4361,6 +4361,17 @@ function gitTimeoutPlan(configured, fallback) { }; } +// Optional historical files may be absent, but a timed-out read says nothing +// about their contents. Let read callers defer/retry instead of caching absence. +export function rethrowGitReadDeadline(error) { + if (error?.code === "FILEGRC_GIT_DEADLINE") throw error; + if (["FILEGRC_HISTORY_DEADLINE", "ETIMEDOUT"].includes(error?.code)) { + const deferred = new Error("Git history could not be read within its time limit.", { cause: error }); + deferred.code = "FILEGRC_GIT_DEADLINE"; + throw deferred; + } +} + function rethrowGitDeadline(error) { if (error?.code === "FILEGRC_GIT_DEADLINE") throw error; } diff --git a/packages/filegrc/src/nonmaterial-amendments.js b/packages/filegrc/src/nonmaterial-amendments.js index e378027..8f07257 100644 --- a/packages/filegrc/src/nonmaterial-amendments.js +++ b/packages/filegrc/src/nonmaterial-amendments.js @@ -10,7 +10,7 @@ import { loadWorkspace } from "./workspace.js"; import { calculateRevision, revisionsMatch } from "./revisions.js"; import { appointmentWasAuthorizedOn } from "./soc2.js"; import { isRfc3339Timestamp, localDateTimeValue } from "./time.js"; -import { getDataRecordHistoryIndex, getFileAtRevision, getFileObjectIdAtRevision, getGitSummary, getWorkingFileObjectId, isDataHistoryAncestor, runGitCommandSync } from "./git.js"; +import { getDataRecordHistoryIndex, getFileAtRevision, getFileObjectIdAtRevision, getGitSummary, getWorkingFileObjectId, isDataHistoryAncestor, runGitCommandSync, rethrowGitReadDeadline } from "./git.js"; const EMAIL = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; const valueAt = (record, path) => String(path || "").split(".").reduce((value, key) => value?.[key], record); @@ -22,14 +22,14 @@ function hasAmendmentRuntimePin(loaded, commit = null) { : readFileSync(join(loaded.root, "package-lock.json"), "utf8"); const engine = JSON.parse(source)?.packages?.["node_modules/filegrc"]; return !engine?.link && ["0.19.x", "0.20.x"].includes(runtimeCompatibilityLine(engine?.version)); - } catch { return false; } + } catch (error) { rethrowGitReadDeadline(error); return false; } } function historicalRecord(loaded, id, commit) { const history = getDataRecordHistoryIndex(loaded.root).historiesById.get(id) || []; const identity = history.find((item) => isDataHistoryAncestor(loaded, item.commit, commit)); if (!identity) return null; - try { return JSON.parse(getFileAtRevision(loaded.root, commit, identity.path)); } catch { return null; } + try { return JSON.parse(getFileAtRevision(loaded.root, commit, identity.path)); } catch (error) { rethrowGitReadDeadline(error); return null; } } // A later source decision on this field ends the old correction chain, even diff --git a/packages/filegrc/src/retention-applications.js b/packages/filegrc/src/retention-applications.js index ca84250..b8aba99 100644 --- a/packages/filegrc/src/retention-applications.js +++ b/packages/filegrc/src/retention-applications.js @@ -13,7 +13,7 @@ import { resourceReviewRevisionsSync, retentionReviewResourceIds, retentionRuleI import { appointmentWasAuthorizedOn, personWasActiveOn } from "./soc2.js"; import { currentPartyPeople } from "./parties.js"; import { createResource } from "./files.js"; -import { getGitSummary, getFileAtRevision, runGitCommandSync, getDataRecordHistoryIndex, isDataHistoryAncestor } from "./git.js"; +import { getGitSummary, getFileAtRevision, runGitCommandSync, rethrowGitReadDeadline, getDataRecordHistoryIndex, isDataHistoryAncestor } from "./git.js"; import { currentCalendarDate } from "./time.js"; const CONFIRMATIONS = ["purposeMatches", "cutoffMatches", "dispositionFeasible", "holdsPreserved", "noConflictingDuties"]; @@ -40,7 +40,7 @@ export function retentionApplicationRuntimeSupported(loaded, commit = null, targ return Boolean(supported(requested) && supported(installed) && lock.packages?.[""]?.dependencies?.filegrc === manifest.dependencies.filegrc && !entry?.link && installed[1] === requested[1] && Number(installed[2]) >= Number(requested[2])); - } catch { return false; } + } catch (error) { rethrowGitReadDeadline(error); return false; } }); } function ruleMarkdown(loaded, rule) { diff --git a/packages/filegrc/test/nonmaterial-amendments.test.js b/packages/filegrc/test/nonmaterial-amendments.test.js index 1fc3019..d327f9e 100644 --- a/packages/filegrc/test/nonmaterial-amendments.test.js +++ b/packages/filegrc/test/nonmaterial-amendments.test.js @@ -1,4 +1,6 @@ import assert from "node:assert/strict"; +import { withCalculationCache } from "../src/calculation-cache.js"; +import { setGitSubprocessObserverForTests, setHistoricalRevisionReadObserverForTests } from "../src/git.js"; import { execFileSync } from "node:child_process"; import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -307,3 +309,24 @@ test("0.20 adoption preserves committed 0.19 amendments and permits unchanged am git(root, "add", "."); git(root, "commit", "-m", "Record subsequent owner label decision"); assert.equal((await validateWorkspace(root)).ok, true); }); + + +test("amendment adoption and historical-source deadlines remain retryable", async (context) => { + const root = await fixture(context), p = await payload(root, "policy-example"); + await saveNonmaterialAmendment(root, p); + git(root, "add", "."); git(root, "commit", "-m", "Record owner correction"); + const loaded = await loadWorkspace(root), amendment = loaded.resources.find(({ id }) => id === p.record.id); + const cache = new WeakMap(), deferred = Object.assign(new Error("Historical read deferred"), { code: "FILEGRC_GIT_DEADLINE" }); + const restorePin = setGitSubprocessObserverForTests(({ args }) => { + if (args[0] === "show" && args[1]?.endsWith(":./package-lock.json")) throw deferred; + }); + try { assert.throws(() => withCalculationCache(cache, () => nonmaterialAmendmentIssues(loaded, amendment)), (error) => error === deferred); } + finally { restorePin(); } + assert.deepEqual(withCalculationCache(cache, () => nonmaterialAmendmentIssues(loaded, amendment)), []); + const restoreSource = setHistoricalRevisionReadObserverForTests(({ relativePath }) => { + if (relativePath === "data/policies/policy-example.json") throw deferred; + }); + try { assert.throws(() => nonmaterialAmendmentIssues(loaded, amendment), (error) => error === deferred); } + finally { restoreSource(); } + assert.deepEqual(nonmaterialAmendmentIssues(loaded, amendment), []); +}); diff --git a/packages/filegrc/test/operating-baseline.test.js b/packages/filegrc/test/operating-baseline.test.js index 864fabd..85f0667 100644 --- a/packages/filegrc/test/operating-baseline.test.js +++ b/packages/filegrc/test/operating-baseline.test.js @@ -1,4 +1,7 @@ import assert from "node:assert/strict"; +import { performance } from "node:perf_hooks"; +import { withGitCommandDeadline } from "../src/git.js"; +import { assessProgramAccessReview, providerAccessControls } from "../src/provider-access-completion.js"; import { execFileSync } from "node:child_process"; import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -546,6 +549,12 @@ test("committed access populations survive later closure while current scope and loaded.resources.push(extraGrant); assert.equal((await baseline("provider-access")).complete, false, "A new active grant requires its own decision and account mapping"); loaded.resources.pop(); revoke(serviceGrant); service.status = "disabled"; + const fresh = { ...loaded, resources: structuredClone(loaded.resources) }; + const freshProgram = fresh.resources.find(({ type }) => type === "program"), freshReview = fresh.resources.find(({ id }) => id === review.id); + const assessFresh = () => assessProgramAccessReview(freshReview, fresh, freshProgram, providerAccessControls(fresh, freshProgram), "2026-10-13"); + assert.throws(() => withGitCommandDeadline(performance.now() - 1, assessFresh), { code: "FILEGRC_GIT_DEADLINE" }, + "Unavailable historical provider context must defer instead of becoming current-population failures"); + assert.equal(assessFresh().complete, true, "The same provider review recovers after its history deadline"); review.populationCount = 1; assert.equal((await baseline("provider-access")).complete, false, "An edited completed review cannot reuse the unchanged committed approval"); }); diff --git a/packages/filegrc/test/retention-applications.test.js b/packages/filegrc/test/retention-applications.test.js index 07d7144..7184430 100644 --- a/packages/filegrc/test/retention-applications.test.js +++ b/packages/filegrc/test/retention-applications.test.js @@ -1,17 +1,22 @@ import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createFilegrcServer } from "../src/server.js"; +import { performance } from "node:perf_hooks"; import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; import { makeRetentionApplicationWorkspace } from "./fixtures/retention-applications.js"; import { commitWorkspaceFiles, writeJson, captureCli } from "./helpers.js"; -import { scaffoldRetentionApplication, saveRetentionApplication, retentionApplicationIssues, retentionApplicationScopeRevision } from "../src/retention-applications.js"; +import { scaffoldRetentionApplication, saveRetentionApplication, retentionApplicationIssues, retentionApplicationScopeRevision, retentionApplicationRuntimeSupported } from "../src/retention-applications.js"; import { assessCollectionReview } from "../src/collection-review.js"; import { assessRetentionReadiness, resourceReviewRevisionsSync, retentionReviewResourceIds } from "../src/retention.js"; import { loadWorkspace } from "../src/workspace.js"; import { validateWorkspace } from "../src/validate.js"; import { createResource, updateResource } from "../src/files.js"; -import { setGitSubprocessObserverForTests } from "../src/git.js"; +import { withCalculationCache } from "../src/calculation-cache.js"; +import { createStateCalculator } from "../src/state-calculator.js"; +import { getGitSummary, withGitCommandDeadline, setGitSubprocessObserverForTests } from "../src/git.js"; import { runCli } from "../src/cli.js"; async function fixture(context) { @@ -372,3 +377,67 @@ test(`ended inventory preserves literal approved retention scope and retired own } }); } + + +test("committed patch upgrades preserve historical adoption after a real deadline and same-cache retry", async (context) => { + const root = await fixture(context); + await saveRetentionApplication(root, await decision(root)); + await commitWorkspaceFiles(root, "Apply existing approved rule"); + const firstCommit = getGitSummary(root).commit; + await writeJson(join(root, "package.json"), { private: true, dependencies: { filegrc: "0.20.6" } }); + await writeJson(join(root, "package-lock.json"), { lockfileVersion: 3, packages: { "": { dependencies: { filegrc: "0.20.6" } }, "node_modules/filegrc": { version: "0.20.6" } } }); + await commitWorkspaceFiles(root, "Adopt compatible patch runtime"); + const loaded = await loadWorkspace(root), application = loaded.resources.find(({ type }) => type === "retention-application"); + const cache = new WeakMap(); + let reachedManifest = false; + const restore = setGitSubprocessObserverForTests(({ args }) => { + if (args[0] === "show" && args[1] === `${firstCommit}:./package.json`) { + reachedManifest = true; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + } + }); + try { + assert.throws(() => withCalculationCache(cache, () => withGitCommandDeadline(performance.now() + 100, + () => retentionApplicationRuntimeSupported(loaded, firstCommit, application))), { code: "FILEGRC_GIT_DEADLINE" }); + assert.equal(reachedManifest, true, "Expire during a historical package read, not before assessment starts"); + } finally { restore(); } + assert.equal(withCalculationCache(cache, () => retentionApplicationRuntimeSupported(loaded, firstCommit, application)), true); + assert.deepEqual(withCalculationCache(cache, () => retentionApplicationIssues(loaded, application)), []); + const calculator = createStateCalculator(); + context.after(() => calculator.close()); + await assert.rejects(calculator.calculate("same-retention-snapshot", loaded, "validation", {}, performance.now() - 1), { code: "FILEGRC_GIT_DEADLINE" }); + const recovered = await calculator.calculate("same-retention-snapshot", loaded, "validation", {}, performance.now() + 20_000); + assert.equal(recovered.ok, true, JSON.stringify(recovered.diagnostics)); + const options = { stateSectionDeadlineMs: 0 }, server = createFilegrcServer(root, options); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + context.after(() => new Promise((resolve) => server.close(resolve))); + const url = `http://127.0.0.1:${server.address().port}`; + const bootstrap = await (await fetch(`${url}/api/state/bootstrap`)).json(); + const path = `${url}/api/state/repository?token=${encodeURIComponent(bootstrap.stateToken)}`; + assert.equal((await fetch(path)).status, 503); + options.stateSectionDeadlineMs = 20_000; + const retry = await fetch(path); + assert.equal(retry.status, 200); + assert.equal((await retry.json()).state.validation.ok, true, "The same browser snapshot must recover without false adoption errors"); + for (const code of ["FILEGRC_HISTORY_DEADLINE", "ETIMEDOUT"]) { + const restoreTimeout = setGitSubprocessObserverForTests(({ args }) => { + if (args[0] === "show" && args[1] === `${firstCommit}:./package-lock.json`) throw Object.assign(new Error("Deferred historical package read"), { code }); + }); + try { assert.throws(() => retentionApplicationRuntimeSupported(loaded, firstCommit, application), { code: "FILEGRC_GIT_DEADLINE" }); } + finally { restoreTimeout(); } + } +}); + +test("a later committed runtime upgrade cannot repair unsupported original adoption", async (context) => { + const root = await fixture(context), preview = await saveRetentionApplication(root, await decision(root), { preview: true }); + await writeJson(join(root, "package.json"), { private: true, dependencies: { filegrc: "0.19.1" } }); + await writeJson(join(root, "package-lock.json"), { lockfileVersion: 3, packages: { "": { dependencies: { filegrc: "0.19.1" } }, "node_modules/filegrc": { version: "0.19.1" } } }); + await mkdir(join(root, "data/retention-applications"), { recursive: true }); + await writeJson(join(root, "data/retention-applications", `${preview.record.id}.json`), preview.record); + await commitWorkspaceFiles(root, "Record unsupported initial adoption"); + await writeJson(join(root, "package.json"), { private: true, dependencies: { filegrc: "0.20.6" } }); + await writeJson(join(root, "package-lock.json"), { lockfileVersion: 3, packages: { "": { dependencies: { filegrc: "0.20.6" } }, "node_modules/filegrc": { version: "0.20.6" } } }); + await commitWorkspaceFiles(root, "Upgrade after the original decision"); + const loaded = await loadWorkspace(root), application = loaded.resources.find(({ type }) => type === "retention-application"); + assert.ok(retentionApplicationIssues(loaded, application).some(({ message }) => message.includes("no later than its first retention application commit"))); +});