Add opt-in setting to automatically include fixtures in migrations on change

__tests__/__snapshots__/settings.test.ts.snap

@@ -22,6 +22,7 @@ exports[`actions is backwards-compatible with untagged command specs 1`] = `
   ],
   "afterCurrent": [],
   "afterReset": [],
+  "autoIncludeFixtures": false,
   "beforeAllMigrations": [],
   "beforeCurrent": [],
   "beforeReset": [],
@@ -63,6 +64,7 @@ exports[`actions parses SQL actions 1`] = `
       "file": "foo.sql",
     },
   ],
+  "autoIncludeFixtures": false,
   "beforeAllMigrations": [],
   "beforeCurrent": [],
   "beforeReset": [],
@@ -99,6 +101,7 @@ exports[`actions parses command actions 1`] = `
   ],
   "afterCurrent": [],
   "afterReset": [],
+  "autoIncludeFixtures": false,
   "beforeAllMigrations": [],
   "beforeCurrent": [],
   "beforeReset": [],
@@ -143,6 +146,7 @@ exports[`actions parses mixed actions 1`] = `
   ],
   "afterCurrent": [],
   "afterReset": [],
+  "autoIncludeFixtures": false,
   "beforeAllMigrations": [],
   "beforeCurrent": [],
   "beforeReset": [],
@@ -184,6 +188,7 @@ exports[`actions parses string values into SQL actions 1`] = `
       "file": "foo.sql",
     },
   ],
+  "autoIncludeFixtures": false,
   "beforeAllMigrations": [],
   "beforeCurrent": [],
   "beforeReset": [],

src/commands/commit.ts

@@ -7,6 +7,7 @@ import {
   readCurrentMigration,
   writeCurrentMigration,
 } from "../current";
+import { clearFixturesCache } from "../fixtures";
 import { calculateHash } from "../hash";
 import { logDbError } from "../instrumentation";
 import {
@@ -100,6 +101,7 @@ export async function _commit(
   await _reset(parsedSettings, true);
   const newMigrationFilepath = `${committedMigrationsFolder}/${newMigrationFilename}`;
   await fsp.writeFile(newMigrationFilepath, finalBody);
+  clearFixturesCache();
   await fsp.chmod(newMigrationFilepath, "440");
 
   parsedSettings.logger.info(

src/commands/uncommit.ts

@@ -7,6 +7,7 @@ import {
   readCurrentMigration,
   writeCurrentMigration,
 } from "../current";
+import { clearFixturesCache } from "../fixtures";
 import {
   getPreviousCommittedMigration,
   parseMigrationText,
@@ -57,6 +58,7 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise<void> {
 
   // Delete the migration from committed and from the DB
   await fsp.unlink(lastMigrationFilepath);
+  clearFixturesCache();
   await undoMigration(parsedSettings, lastMigration);
 
   parsedSettings.logger.info(

src/commands/watch.ts

@@ -3,6 +3,7 @@ import { CommandModule } from "yargs";
 
 import { getCurrentMigrationLocation, writeCurrentMigration } from "../current";
 import { makeCurrentMigrationRunner } from "../currentRunner";
+import { clearFixturesCache } from "../fixtures";
 import { isLoggedError } from "../lib";
 import { ParsedSettings, parseSettings, Settings } from "../settings";
 import { CommonArgv, getSettings } from "./_common";
@@ -90,6 +91,17 @@ export async function _watch(
     watcher.on("change", queue);
     watcher.on("unlink", queue);
     watcher.once("ready", queue);
+
+    // When you change branch, clear the fixtures cache
+    const committedWatcher = chokidar.watch(
+      [`${parsedSettings.migrationsFolder}/committed`],
+      CHOKIDAR_OPTIONS,
+    );
+    committedWatcher.on("add", clearFixturesCache);
+    committedWatcher.on("change", clearFixturesCache);
+    committedWatcher.on("unlink", clearFixturesCache);
+    committedWatcher.once("ready", clearFixturesCache);
+
     return Promise.resolve();
   }
 }

src/current.ts

@@ -1,6 +1,7 @@
 import * as assert from "assert";
 import { promises as fsp } from "fs";
 
+import { getCommittedFixturesState, getLocalFixtures } from "./fixtures";
 import { isNoTransactionDefined } from "./header";
 import { readFileOrError, readFileOrNull, statOrNull } from "./lib";
 import {
@@ -57,7 +58,48 @@ export async function readCurrentMigration(
   location: CurrentMigrationLocation,
 ): Promise<{ raw: string; compiled: string }> {
   const { body: raw, paths } = await readCurrentMigrationRaw(location);
-  const compiled = await compileIncludes(parsedSettings, raw, new Set(paths));
+  let compiled = raw;
+  if (parsedSettings.autoIncludeFixtures) {
+    const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm;
+    const alreadyIncluded = [...compiled.matchAll(regex)].map(
+      (match) => match[1],
+    );
+
+    // Find any fixture that has changed since the last commit that hasn't been
+    // specifically `--!include`'d
+    const committedState = await getCommittedFixturesState(parsedSettings);
+    const currentFixtures = await getLocalFixtures(parsedSettings);
+    const changedFixtures: string[] = [];
+    for (const [relativePath, { hash, hasNoTransaction }] of Object.entries(
+      currentFixtures,
+    )) {
+      if (committedState[relativePath] !== hash) {
+        // TODO: maybe be a bit more flexible about matches here: `./`, etc
+        if (!alreadyIncluded.includes(relativePath)) {
+          if (hasNoTransaction) {
+            parsedSettings.logger.warn(
+              `Not including modified fixture ${relativePath} because it includes --! no-transaction`,
+              { relativePath, noTransaction: true },
+            );
+          } else {
+            changedFixtures.push(relativePath);
+          }
+        }
+      }
+    }
+
+    // Prefix the migration with the changed fixture includes
+    if (changedFixtures.length > 0) {
+      changedFixtures.sort((a, z) => (a < z ? -1 : a > z ? 1 : 0));
+      compiled =
+        changedFixtures
+          .map((relativePath) => `--!include ${relativePath}`)
+          .join("\n") +
+        "\n\n--!endAuto\n\n" +
+        compiled;
+    }
+  }
+  compiled = await compileIncludes(parsedSettings, compiled, new Set(paths));
   return { raw, compiled };
 }
 
@@ -152,8 +194,10 @@ async function readCurrentMigrationRaw(
 export async function writeCurrentMigration(
   parsedSettings: ParsedSettings,
   location: CurrentMigrationLocation,
-  body: string,
+  rawBody: string,
 ): Promise<void> {
+  // Strip the auto-included fixtures
+  const body = rawBody.replace(/^[\s\S]*\n--!endAuto\n\n/, "");
   if (body.trim() + "\n" !== body) {
     throw new Error(
       "graphile-migrate error - 'body' should be sanitized before being passed to 'writeCurrentMigration'",

src/fixtures.ts

@@ -0,0 +1,129 @@
+import * as crypto from "crypto";
+import { promises as fsp } from "fs";
+import { sep } from "path";
+import pgMinify from "pg-minify";
+
+import { isNoTransactionDefined } from "./header";
+import { realpathOrNull, walk } from "./lib";
+import { compileIncludes, getAllMigrations } from "./migration";
+import type { ParsedSettings } from "./settings";
+
+interface FileFixture {
+  /**
+   * The full path to this fixture on disk
+   */
+  fullPath: string;
+
+  /**
+   * The path to this fixture relative to the fixtures folder
+   */
+  relativePath: string;
+
+  /**
+   * A hash of the content of the fixture
+   */
+  hash: string;
+
+  hasNoTransaction: boolean;
+}
+
+interface FixturesCache {
+  fixtures: {
+    [relativePath: string]: Array<{
+      filename: string;
+      hash: string;
+    }>;
+  };
+}
+
+function hashFixture(fixtureText: string) {
+  const minified = pgMinify(fixtureText);
+  return crypto
+    .createHash("sha1")
+    .update(minified.trim() + "\n")
+    .digest("hex");
+}
+
+export async function getLocalFixtures(
+  parsedSettings: ParsedSettings,
+): Promise<{ [relativePath: string]: FileFixture }> {
+  const allFixtures = Object.create(null) as {
+    [relativePath: string]: FileFixture;
+  };
+  const rawFixturesPath = `${parsedSettings.migrationsFolder}/fixtures`;
+  const fixturesPath = await realpathOrNull(rawFixturesPath);
+  if (fixturesPath) {
+    await walk(fixturesPath, async (relativePath, ent) => {
+      if (ent.name.startsWith(".") || !ent.name.endsWith(".sql")) return;
+      const fullPath = ent.parentPath + sep + ent.name;
+      const contents = await fsp.readFile(fullPath, "utf8");
+      const compiled = await compileIncludes(
+        parsedSettings,
+        contents,
+        new Set([fullPath]),
+      );
+      allFixtures[relativePath] = {
+        fullPath,
+        relativePath,
+        hash: hashFixture(compiled),
+        hasNoTransaction: isNoTransactionDefined(compiled),
+      };
+    });
+  }
+  return allFixtures;
+}
+
+let committedFixturesCache: [ParsedSettings, Promise<FixturesCache>] | null =
+  null;
+export function clearFixturesCache() {
+  committedFixturesCache = null;
+}
+
+export function getCommittedFixturesCache(
+  parsedSettings: ParsedSettings,
+): Promise<FixturesCache> {
+  if (committedFixturesCache?.[0] === parsedSettings) {
+    return committedFixturesCache[1];
+  }
+  const promise = _reallyGetCommittedFixturesCache(parsedSettings);
+  committedFixturesCache = [parsedSettings, promise];
+  return promise;
+}
+
+async function _reallyGetCommittedFixturesCache(
+  parsedSettings: ParsedSettings,
+): Promise<FixturesCache> {
+  const cache: FixturesCache = {
+    fixtures: Object.create(null) as Record<string, never>,
+  };
+  const allMigrations = await getAllMigrations(parsedSettings);
+  for (const migration of allMigrations) {
+    const { filename, body } = migration;
+    const re = /(^|\n)--! Included ([^\r\n]+)\r?\n/g;
+    let match: RegExpExecArray | null;
+    while ((match = re.exec(body)) != null) {
+      const [, relativePath] = match;
+      const i = body.indexOf(`--! EndIncluded ${relativePath}`, match.index);
+      if (i < 0) throw new Error(`Found Included with no EndIncluded`);
+      const content = body.substring(match.index + match[0].length, i).trim();
+      cache.fixtures[relativePath] ??= [];
+      const hash = hashFixture(content);
+      cache.fixtures[relativePath].push({ filename, hash });
+    }
+  }
+  return cache;
+}
+
+export async function getCommittedFixturesState(
+  parsedSettings: ParsedSettings,
+): Promise<{ [relativePath: string]: string }> {
+  const cache = await getCommittedFixturesCache(parsedSettings);
+  const state = Object.create(null) as { [relativePath: string]: string };
+  for (const relativePath in cache.fixtures) {
+    const hash = cache.fixtures[relativePath].at(-1)?.hash;
+    if (hash != null) {
+      state[relativePath] = hash;
+    }
+  }
+  return state;
+}

src/lib.ts

@@ -1,4 +1,6 @@
-import { promises as fsp, Stats } from "fs";
+import type { Dirent, Stats } from "fs";
+import * as fsp from "fs/promises";
+import { sep } from "path";
 
 export function mergeWithoutClobbering(
   original: { [key: string]: string | undefined },
@@ -79,3 +81,39 @@ export async function realpathOrNull(path: string): Promise<string | null> {
     return null;
   }
 }
+
+export async function walk(
+  root: string,
+  callback: (relativePath: string, ent: Dirent<string>) => Promise<void>,
+): Promise<void> {
+  return _walk(
+    root,
+    [],
+    { walkHiddenFolders: false, callbackHiddenFiles: false },
+    callback,
+  );
+}
+
+async function _walk(
+  root: string,
+  paths: readonly string[],
+  options: {
+    walkHiddenFolders: boolean;
+    callbackHiddenFiles: boolean;
+  },
+  callback: (relativePath: string, ent: Dirent<string>) => Promise<void>,
+): Promise<void> {
+  const directory = [root, ...paths].join(sep);
+  const entries = await fsp.readdir(directory, { withFileTypes: true });
+  await Promise.all(
+    entries.map(async (entry) => {
+      if (entry.isDirectory()) {
+        if (!options.walkHiddenFolders && entry.name.startsWith(".")) return;
+        await _walk(root, [...paths, entry.name], options, callback);
+      } else {
+        if (!options.callbackHiddenFiles && entry.name.startsWith(".")) return;
+        await callback([...paths, entry.name].join("/"), entry);
+      }
+    }),
+  );
+}

src/settings.ts

@@ -84,6 +84,19 @@ export interface Settings {
   afterCurrent?: Actions;
   blankMigrationContent?: string;
   logger?: Logger;
+  /**
+   * On `commit` (or when using `watch` mode), find fixtures that have changed
+   * since the last commit and include them in this migration.
+   *
+   * NOTE: in this mode, fixtures should only be used for stateless database
+   * entities: functions, RLS policies, grants, views, etc - things that can be
+   * dropped and recreated almost instantly. You should be very wary of using
+   * them with stateful entities such as indexes, constraints and materialized
+   * views as dropping and recreating these takes significant time.
+   *
+   * @experimental
+   */
+  autoIncludeFixtures?: boolean;
 }
 
 // NOTE: only override values that differ (e.g. changing non-nullability)
@@ -102,6 +115,8 @@ export interface ParsedSettings extends Settings {
   afterCurrent: ActionSpec[];
   blankMigrationContent: string;
   logger: Logger;
+  /** @experimental */
+  autoIncludeFixtures: boolean;
 }
 
 export async function parseSettings(
@@ -289,6 +304,15 @@ export async function parseSettings(
     },
   );
 
+  const autoIncludeFixtures = await check(
+    "autoIncludeFixtures",
+    (rawAIF): boolean => {
+      if (typeof rawAIF === "boolean") return rawAIF;
+      if (rawAIF == null) return false;
+      throw new Error(`Invalid value for autoIncludeFixtures`);
+    },
+  );
+
   const validateAction = makeValidateActionCallback(logger);
   const rootValidateAction = makeValidateActionCallback(logger, true);
 
@@ -390,6 +414,7 @@ export async function parseSettings(
     placeholders,
     blankMigrationContent,
     logger,
+    autoIncludeFixtures,
   };
 }