diff --git a/__tests__/__snapshots__/settings.test.ts.snap b/__tests__/__snapshots__/settings.test.ts.snap index 68d7da5..d438b85 100644 --- a/__tests__/__snapshots__/settings.test.ts.snap +++ b/__tests__/__snapshots__/settings.test.ts.snap @@ -22,6 +22,7 @@ exports[`actions is backwards-compatible with untagged command specs 1`] = ` ], "afterCurrent": [], "afterReset": [], + "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -63,6 +64,7 @@ exports[`actions parses SQL actions 1`] = ` "file": "foo.sql", }, ], + "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -99,6 +101,7 @@ exports[`actions parses command actions 1`] = ` ], "afterCurrent": [], "afterReset": [], + "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -143,6 +146,7 @@ exports[`actions parses mixed actions 1`] = ` ], "afterCurrent": [], "afterReset": [], + "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -184,6 +188,7 @@ exports[`actions parses string values into SQL actions 1`] = ` "file": "foo.sql", }, ], + "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], diff --git a/src/commands/commit.ts b/src/commands/commit.ts index e5b57ff..7d36758 100644 --- a/src/commands/commit.ts +++ b/src/commands/commit.ts @@ -7,6 +7,7 @@ import { readCurrentMigration, writeCurrentMigration, } from "../current"; +import { clearFixturesCache } from "../fixtures"; import { calculateHash } from "../hash"; import { logDbError } from "../instrumentation"; import { @@ -100,6 +101,7 @@ export async function _commit( await _reset(parsedSettings, true); const newMigrationFilepath = `${committedMigrationsFolder}/${newMigrationFilename}`; await fsp.writeFile(newMigrationFilepath, finalBody); + clearFixturesCache(); await fsp.chmod(newMigrationFilepath, "440"); parsedSettings.logger.info( diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 39b23ee..d5aa27c 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -7,6 +7,7 @@ import { readCurrentMigration, writeCurrentMigration, } from "../current"; +import { clearFixturesCache } from "../fixtures"; import { getPreviousCommittedMigration, parseMigrationText, @@ -57,6 +58,7 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { // Delete the migration from committed and from the DB await fsp.unlink(lastMigrationFilepath); + clearFixturesCache(); await undoMigration(parsedSettings, lastMigration); parsedSettings.logger.info( diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 6c6bdaa..bea497a 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -3,6 +3,7 @@ import { CommandModule } from "yargs"; import { getCurrentMigrationLocation, writeCurrentMigration } from "../current"; import { makeCurrentMigrationRunner } from "../currentRunner"; +import { clearFixturesCache } from "../fixtures"; import { isLoggedError } from "../lib"; import { ParsedSettings, parseSettings, Settings } from "../settings"; import { CommonArgv, getSettings } from "./_common"; @@ -90,6 +91,17 @@ export async function _watch( watcher.on("change", queue); watcher.on("unlink", queue); watcher.once("ready", queue); + + // When you change branch, clear the fixtures cache + const committedWatcher = chokidar.watch( + [`${parsedSettings.migrationsFolder}/committed`], + CHOKIDAR_OPTIONS, + ); + committedWatcher.on("add", clearFixturesCache); + committedWatcher.on("change", clearFixturesCache); + committedWatcher.on("unlink", clearFixturesCache); + committedWatcher.once("ready", clearFixturesCache); + return Promise.resolve(); } } diff --git a/src/current.ts b/src/current.ts index 85acb08..8d0228d 100644 --- a/src/current.ts +++ b/src/current.ts @@ -1,6 +1,7 @@ import * as assert from "assert"; import { promises as fsp } from "fs"; +import { getCommittedFixturesState, getLocalFixtures } from "./fixtures"; import { isNoTransactionDefined } from "./header"; import { readFileOrError, readFileOrNull, statOrNull } from "./lib"; import { @@ -57,7 +58,48 @@ export async function readCurrentMigration( location: CurrentMigrationLocation, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); - const compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); + let compiled = raw; + if (parsedSettings.autoIncludeFixtures) { + const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; + const alreadyIncluded = [...compiled.matchAll(regex)].map( + (match) => match[1], + ); + + // Find any fixture that has changed since the last commit that hasn't been + // specifically `--!include`'d + const committedState = await getCommittedFixturesState(parsedSettings); + const currentFixtures = await getLocalFixtures(parsedSettings); + const changedFixtures: string[] = []; + for (const [relativePath, { hash, hasNoTransaction }] of Object.entries( + currentFixtures, + )) { + if (committedState[relativePath] !== hash) { + // TODO: maybe be a bit more flexible about matches here: `./`, etc + if (!alreadyIncluded.includes(relativePath)) { + if (hasNoTransaction) { + parsedSettings.logger.warn( + `Not including modified fixture ${relativePath} because it includes --! no-transaction`, + { relativePath, noTransaction: true }, + ); + } else { + changedFixtures.push(relativePath); + } + } + } + } + + // Prefix the migration with the changed fixture includes + if (changedFixtures.length > 0) { + changedFixtures.sort((a, z) => (a < z ? -1 : a > z ? 1 : 0)); + compiled = + changedFixtures + .map((relativePath) => `--!include ${relativePath}`) + .join("\n") + + "\n\n--!endAuto\n\n" + + compiled; + } + } + compiled = await compileIncludes(parsedSettings, compiled, new Set(paths)); return { raw, compiled }; } @@ -152,8 +194,10 @@ async function readCurrentMigrationRaw( export async function writeCurrentMigration( parsedSettings: ParsedSettings, location: CurrentMigrationLocation, - body: string, + rawBody: string, ): Promise { + // Strip the auto-included fixtures + const body = rawBody.replace(/^[\s\S]*\n--!endAuto\n\n/, ""); if (body.trim() + "\n" !== body) { throw new Error( "graphile-migrate error - 'body' should be sanitized before being passed to 'writeCurrentMigration'", diff --git a/src/fixtures.ts b/src/fixtures.ts new file mode 100644 index 0000000..7015d82 --- /dev/null +++ b/src/fixtures.ts @@ -0,0 +1,129 @@ +import * as crypto from "crypto"; +import { promises as fsp } from "fs"; +import { sep } from "path"; +import pgMinify from "pg-minify"; + +import { isNoTransactionDefined } from "./header"; +import { realpathOrNull, walk } from "./lib"; +import { compileIncludes, getAllMigrations } from "./migration"; +import type { ParsedSettings } from "./settings"; + +interface FileFixture { + /** + * The full path to this fixture on disk + */ + fullPath: string; + + /** + * The path to this fixture relative to the fixtures folder + */ + relativePath: string; + + /** + * A hash of the content of the fixture + */ + hash: string; + + hasNoTransaction: boolean; +} + +interface FixturesCache { + fixtures: { + [relativePath: string]: Array<{ + filename: string; + hash: string; + }>; + }; +} + +function hashFixture(fixtureText: string) { + const minified = pgMinify(fixtureText); + return crypto + .createHash("sha1") + .update(minified.trim() + "\n") + .digest("hex"); +} + +export async function getLocalFixtures( + parsedSettings: ParsedSettings, +): Promise<{ [relativePath: string]: FileFixture }> { + const allFixtures = Object.create(null) as { + [relativePath: string]: FileFixture; + }; + const rawFixturesPath = `${parsedSettings.migrationsFolder}/fixtures`; + const fixturesPath = await realpathOrNull(rawFixturesPath); + if (fixturesPath) { + await walk(fixturesPath, async (relativePath, ent) => { + if (ent.name.startsWith(".") || !ent.name.endsWith(".sql")) return; + const fullPath = ent.parentPath + sep + ent.name; + const contents = await fsp.readFile(fullPath, "utf8"); + const compiled = await compileIncludes( + parsedSettings, + contents, + new Set([fullPath]), + ); + allFixtures[relativePath] = { + fullPath, + relativePath, + hash: hashFixture(compiled), + hasNoTransaction: isNoTransactionDefined(compiled), + }; + }); + } + return allFixtures; +} + +let committedFixturesCache: [ParsedSettings, Promise] | null = + null; +export function clearFixturesCache() { + committedFixturesCache = null; +} + +export function getCommittedFixturesCache( + parsedSettings: ParsedSettings, +): Promise { + if (committedFixturesCache?.[0] === parsedSettings) { + return committedFixturesCache[1]; + } + const promise = _reallyGetCommittedFixturesCache(parsedSettings); + committedFixturesCache = [parsedSettings, promise]; + return promise; +} + +async function _reallyGetCommittedFixturesCache( + parsedSettings: ParsedSettings, +): Promise { + const cache: FixturesCache = { + fixtures: Object.create(null) as Record, + }; + const allMigrations = await getAllMigrations(parsedSettings); + for (const migration of allMigrations) { + const { filename, body } = migration; + const re = /(^|\n)--! Included ([^\r\n]+)\r?\n/g; + let match: RegExpExecArray | null; + while ((match = re.exec(body)) != null) { + const [, relativePath] = match; + const i = body.indexOf(`--! EndIncluded ${relativePath}`, match.index); + if (i < 0) throw new Error(`Found Included with no EndIncluded`); + const content = body.substring(match.index + match[0].length, i).trim(); + cache.fixtures[relativePath] ??= []; + const hash = hashFixture(content); + cache.fixtures[relativePath].push({ filename, hash }); + } + } + return cache; +} + +export async function getCommittedFixturesState( + parsedSettings: ParsedSettings, +): Promise<{ [relativePath: string]: string }> { + const cache = await getCommittedFixturesCache(parsedSettings); + const state = Object.create(null) as { [relativePath: string]: string }; + for (const relativePath in cache.fixtures) { + const hash = cache.fixtures[relativePath].at(-1)?.hash; + if (hash != null) { + state[relativePath] = hash; + } + } + return state; +} diff --git a/src/lib.ts b/src/lib.ts index 1c64db0..8cdbc5d 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -1,4 +1,6 @@ -import { promises as fsp, Stats } from "fs"; +import type { Dirent, Stats } from "fs"; +import * as fsp from "fs/promises"; +import { sep } from "path"; export function mergeWithoutClobbering( original: { [key: string]: string | undefined }, @@ -79,3 +81,39 @@ export async function realpathOrNull(path: string): Promise { return null; } } + +export async function walk( + root: string, + callback: (relativePath: string, ent: Dirent) => Promise, +): Promise { + return _walk( + root, + [], + { walkHiddenFolders: false, callbackHiddenFiles: false }, + callback, + ); +} + +async function _walk( + root: string, + paths: readonly string[], + options: { + walkHiddenFolders: boolean; + callbackHiddenFiles: boolean; + }, + callback: (relativePath: string, ent: Dirent) => Promise, +): Promise { + const directory = [root, ...paths].join(sep); + const entries = await fsp.readdir(directory, { withFileTypes: true }); + await Promise.all( + entries.map(async (entry) => { + if (entry.isDirectory()) { + if (!options.walkHiddenFolders && entry.name.startsWith(".")) return; + await _walk(root, [...paths, entry.name], options, callback); + } else { + if (!options.callbackHiddenFiles && entry.name.startsWith(".")) return; + await callback([...paths, entry.name].join("/"), entry); + } + }), + ); +} diff --git a/src/settings.ts b/src/settings.ts index 79e665e..0f2af83 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -84,6 +84,19 @@ export interface Settings { afterCurrent?: Actions; blankMigrationContent?: string; logger?: Logger; + /** + * On `commit` (or when using `watch` mode), find fixtures that have changed + * since the last commit and include them in this migration. + * + * NOTE: in this mode, fixtures should only be used for stateless database + * entities: functions, RLS policies, grants, views, etc - things that can be + * dropped and recreated almost instantly. You should be very wary of using + * them with stateful entities such as indexes, constraints and materialized + * views as dropping and recreating these takes significant time. + * + * @experimental + */ + autoIncludeFixtures?: boolean; } // NOTE: only override values that differ (e.g. changing non-nullability) @@ -102,6 +115,8 @@ export interface ParsedSettings extends Settings { afterCurrent: ActionSpec[]; blankMigrationContent: string; logger: Logger; + /** @experimental */ + autoIncludeFixtures: boolean; } export async function parseSettings( @@ -289,6 +304,15 @@ export async function parseSettings( }, ); + const autoIncludeFixtures = await check( + "autoIncludeFixtures", + (rawAIF): boolean => { + if (typeof rawAIF === "boolean") return rawAIF; + if (rawAIF == null) return false; + throw new Error(`Invalid value for autoIncludeFixtures`); + }, + ); + const validateAction = makeValidateActionCallback(logger); const rootValidateAction = makeValidateActionCallback(logger, true); @@ -390,6 +414,7 @@ export async function parseSettings( placeholders, blankMigrationContent, logger, + autoIncludeFixtures, }; }