refactor(core): single Record-production module owns file→Record and path classification

docs/glossary.md

@@ -47,6 +47,13 @@ When GraphQL is your **query interface**, the **generated GraphQL schema** descr
 
 **One loaded item** in a collection: the structured result of reading a file (metadata, body, derived fields) that your app treats as a single unit. “Record” here means **a document-shaped object in memory**, not a row in a remote database.
 
+### Record production
+
+The transformation of collection-grouped source files into records, with core
+stamping source context (`_path`, `_filename`) last. Record production excludes
+validation and path ownership, which are handled by `validateRecords` and
+`classifyPath`.
+
 ### Relation
 
 A **configured link** from entries in one collection to another (for example, `refs` in config mapping a post field to an `Author` collection). Relations express **associations between flat-file content**, not foreign keys managed by a separate database server.

packages/core/src/export/json.ts

@@ -1,14 +1,12 @@
-import { VFile } from 'vfile';
 import { relative } from 'node:path';
-import { generateSchema } from '../generators/schema';
 import {
   ConfigResult,
   ContentEntry,
   EntryNode,
   LoadedFlatbreadConfig,
-  Transformer,
 } from '../types';
 import { normalizeIdentifier } from '../utils/ids';
+import { produceRecords, validateRecords } from '../records';
 
 export interface JsonExportOptions {
   collections?: readonly string[];
@@ -26,7 +24,7 @@ export type JsonExportResult = Record<string, EntryNode[]>;
  * - record IDs and configured relation fields are normalized with the same ID
  *   semantics used by query resolvers;
  * - `_path` is emitted relative to `pathRoot` (default: `process.cwd()`);
- * - invalid IDs/refs fail through the same validation gate as schema
+ * - invalid IDs/refs fail through the same record validation seam as schema
  *   generation before export output is returned.
  */
 export async function exportCollectionsAsJson(
@@ -38,13 +36,10 @@ export async function exportCollectionsAsJson(
     throw new Error('Config is not defined');
   }
 
-  // Reuse schema generation as the validation gate so exports cannot silently
-  // serialize broken ids or refs.
-  await generateSchema(configResult);
-
   config.source.initialize?.(config);
   const rawNodes = await config.source.fetch(config.content);
-  const transformerByExtension = getTransformerExtensionMap(config.transformer);
+  const produced = produceRecords(rawNodes, config);
+  validateRecords(produced, config);
   const selected = new Set(
     options.collections ?? config.content.map((entry) => entry.collection)
   );
@@ -63,12 +58,11 @@ export async function exportCollectionsAsJson(
   }
   const result: JsonExportResult = {};
 
-  for (const [collection, nodes] of Object.entries(rawNodes)) {
+  for (const [collection, nodes] of Object.entries(produced)) {
     if (!selected.has(collection)) continue;
 
     const contentEntry = contentByCollection.get(collection);
     const records = nodes
-      .map((node) => parseNode(node, transformerByExtension))
       .map((entry) =>
         normalizeRecord(entry, contentEntry, options.pathRoot ?? process.cwd())
       )
@@ -89,34 +83,6 @@ export async function exportCollectionsAsJson(
   );
 }
 
-function getTransformerExtensionMap(
-  transformer: Transformer[]
-): Map<string, Transformer> {
-  const transformerMap = new Map<string, Transformer>();
-  transformer.forEach((nextTransformer) => {
-    nextTransformer.extensions.forEach((extension) => {
-      transformerMap.set(extension, nextTransformer);
-    });
-  });
-  return transformerMap;
-}
-
-function parseNode(
-  node: VFile,
-  transformerByExtension: Map<string, Transformer>
-): EntryNode {
-  const transformer = transformerByExtension.get(node.extname ?? '');
-  if (!transformer?.parse) {
-    throw new Error(`no transformer found for ${node.path}`);
-  }
-
-  return {
-    ...transformer.parse(node),
-    _path: node.path,
-    _filename: node.basename,
-  };
-}
-
 function normalizeRecord(
   entry: EntryNode,
   contentEntry: ContentEntry | undefined,

packages/core/src/export/tests/json.test.ts

@@ -3,6 +3,7 @@ import filesystem from '@flatbread/source-filesystem';
 import markdownTransformer from '@flatbread/transformer-markdown';
 import { exportCollectionsAsJson } from '../json';
 import { initializeConfig } from '../../utils/initializeConfig';
+import { VFile } from 'vfile';
 
 test('exports selected collections as stable normalized JSON', async (t) => {
   const config = initializeConfig({
@@ -90,3 +91,27 @@ test('reuses validation diagnostics before exporting JSON', async (t) => {
 
   t.regex(error?.message ?? '', /Author record id "123" is duplicated/);
 });
+
+test('fetches content exactly once per export', async (t) => {
+  const events: string[] = [];
+  const config = initializeConfig({
+    source: {
+      initialize: () => events.push('initialize'),
+      fetch: async () => {
+        events.push('fetch');
+        return { Post: [new VFile({ path: 'post.md', value: 'id: post' })] };
+      },
+    },
+    transformer: {
+      extensions: ['.md'],
+      inspect: String,
+      parse: () => ({ id: 'post' }),
+    },
+    content: [{ collection: 'Post', path: 'content' }],
+  });
+
+  const result = await exportCollectionsAsJson({ config });
+  t.is(events.filter((event) => event === 'fetch').length, 1);
+  t.deepEqual(events.slice(0, 2), ['initialize', 'fetch']);
+  t.is(result.Post[0].id, 'post');
+});

packages/core/src/generators/contentGraph.ts

@@ -1,4 +1,4 @@
-import { extname, resolve } from 'node:path';
+import { resolve } from 'node:path';
 import type { VFile } from 'vfile';
 import type {
   ContentGraphSnapshot,
@@ -8,11 +8,7 @@ import type {
   LoadedFlatbreadConfig,
 } from '../types';
 import { getNodeIdentifier, normalizeIdentifier } from '../utils/ids';
-import { validateCollectionReferences } from '../utils/references';
-import {
-  optionallyTransformContentNodes,
-  validateCollectionIdentifiers,
-} from './schema';
+import { classifyPath, produceRecords, validateRecords } from '../records';
 
 const keyFor = (collection: string, id: string) => `${collection}\u0000${id}`;
 
@@ -21,9 +17,8 @@ export async function buildContentGraph(
 ): Promise<ContentGraphSnapshot> {
   config.source.initialize?.(config);
   const fetched = await config.source.fetch(config.content);
-  const transformed = optionallyTransformContentNodes(fetched, config);
-  const nodesByCollection = validateCollectionIdentifiers(transformed);
-  validateCollectionReferences(transformed, config.content);
+  const produced = produceRecords(fetched, config);
+  const nodesByCollection = validateRecords(produced, config);
   return makeSnapshot(config, nodesByCollection);
 }
 
@@ -102,8 +97,7 @@ export async function patchContentGraph(
       indexed.node as ContentNode
     );
   }
-  validateCollectionIdentifiers(nodesByCollection);
-  validateCollectionReferences(nodesByCollection, previous.config.content);
+  validateRecords(nodesByCollection, previous.config);
   return makeSnapshot(previous.config, nodesByCollection);
 }
 
@@ -113,10 +107,13 @@ function transformFiles(
 ): Map<string, IndexedContentNode> {
   const grouped: Record<string, VFile[]> = {};
   for (const file of files) {
-    const collection = collectionForPath(resolve(file.path), config);
-    if (collection) (grouped[collection] ??= []).push(file);
+    const classification = classifyPath(resolve(file.path), config);
+    if (classification) {
+      file.data = { ...file.data, ...classification.captures };
+      (grouped[classification.collection] ??= []).push(file);
+    }
   }
-  const transformed = optionallyTransformContentNodes(grouped, config);
+  const transformed = produceRecords(grouped, config);
   const result = new Map<string, IndexedContentNode>();
   for (const [collection, nodes] of Object.entries(transformed)) {
     for (const node of nodes) {
@@ -187,36 +184,3 @@ function referencesFor(
   }
   return result;
 }
-
-function collectionForPath(
-  path: string,
-  config: LoadedFlatbreadConfig
-): string | undefined {
-  const extension = extname(path).toLowerCase();
-  if (
-    !(config.loaded.extensions ?? []).some(
-      (item) => `.${item.replace(/^\./, '')}`.toLowerCase() === extension
-    )
-  ) {
-    return undefined;
-  }
-  return config.content.find((entry) => {
-    if (!entry.path) return false;
-    const root = resolve(entry.path);
-    if (path === root || path.startsWith(`${root}/`)) return true;
-    return pathPatternMatches(path, root);
-  })?.collection;
-}
-
-function pathPatternMatches(path: string, pattern: string): boolean {
-  const escaped = pattern
-    .split('/')
-    .map((segment) => {
-      if (/^\[[^\]]+\]/.test(segment)) {
-        return '[^/]+';
-      }
-      return segment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-    })
-    .join('/');
-  return new RegExp(`^${escaped}(?:/|$)`).test(path);
-}

packages/core/src/generators/schema.ts

@@ -2,7 +2,6 @@ import { schemaComposer } from 'graphql-compose';
 import { composeWithJson } from 'graphql-compose-json';
 import { cloneDeep, merge } from 'lodash-es';
 import plur from 'plur';
-import { VFile } from 'vfile';
 import { cacheSchema, checkCacheForSchema } from '../cache/cache';
 import {
   generateArgsForAllItemQuery,
@@ -16,15 +15,13 @@ import {
   ContentNode,
   EntryNode,
   LoadedFlatbreadConfig,
-  Transformer,
 } from '../types';
-import { map } from '../utils/map';
 import {
   getNodeIdentifier,
   normalizeIdentifier,
   normalizeOptionalIdentifier,
 } from '../utils/ids';
-import { validateCollectionReferences } from '../utils/references';
+import { produceRecords, validateRecords } from '../records';
 import { generateCollection } from './generateCollection';
 
 interface RootQueries {
@@ -37,7 +34,8 @@ interface ResolverPayload {
 }
 
 /**
- * Generates a GraphQL schema from content nodes.
+ * Generates a GraphQL schema from content nodes. A supplied content graph is
+ * trusted as already validated by buildContentGraph or patchContentGraph.
  *
  * @param configResult the result of the config file processing
  */
@@ -61,13 +59,8 @@ export async function generateSchema(
   } else {
     config.source.initialize?.(config);
     const allContentNodes = await config.source.fetch(config.content);
-    allContentNodesJSON = optionallyTransformContentNodes(
-      allContentNodes,
-      config
-    );
-    contentNodesByCollection =
-      validateCollectionIdentifiers(allContentNodesJSON);
-    validateCollectionReferences(allContentNodesJSON, config.content);
+    allContentNodesJSON = produceRecords(allContentNodes, config);
+    contentNodesByCollection = validateRecords(allContentNodesJSON, config);
   }
 
   // Content validation must run before returning a cached schema because the
@@ -288,104 +281,3 @@ const fetchPreknownSchemaFragments = (
     {}
   );
 };
-
-export function validateCollectionIdentifiers(
-  allContentNodesJSON: Record<string, EntryNode[]>
-): Record<string, ContentNode[]> {
-  const errors: string[] = [];
-  const contentNodesByCollection: Record<string, ContentNode[]> = {};
-
-  Object.entries(allContentNodesJSON).forEach(([collection, nodes]) => {
-    const seen = new Map<string, EntryNode>();
-    contentNodesByCollection[collection] = [];
-
-    nodes.forEach((node) => {
-      try {
-        const normalizedId = getNodeIdentifier(node, collection);
-        const existing = seen.get(normalizedId);
-
-        if (existing) {
-          errors.push(
-            `${collection} record id "${normalizedId}" is duplicated after normalization${sourceContext(
-              existing
-            )}${sourceContext(node)}`
-          );
-        } else {
-          seen.set(normalizedId, node);
-        }
-        contentNodesByCollection[collection].push(node as ContentNode);
-      } catch (error) {
-        errors.push(error instanceof Error ? error.message : String(error));
-      }
-    });
-  });
-
-  if (errors.length > 0) {
-    errors.sort();
-    throw new Error(
-      `Flatbread found ${errors.length} invalid record ID${
-        errors.length === 1 ? '' : 's'
-      }:\n${errors.map((message) => `- ${message}`).join('\n')}`
-    );
-  }
-
-  return contentNodesByCollection;
-}
-
-function sourceContext(node: EntryNode): string {
-  return typeof node._path === 'string' ? ` (${node._path})` : '';
-}
-
-function getTransformerExtensionMap(
-  transformer: Transformer[]
-): Map<string, Transformer> {
-  const transformerMap = new Map<string, Transformer>();
-  transformer.forEach((t) => {
-    t.extensions.forEach((extension) => {
-      transformerMap.set(extension, t);
-    });
-  });
-  return transformerMap;
-}
-
-/**
- * Transforms the content nodes to the expected JSON format. If no transformer is defined, the content nodes are returned as is.
- *
- * @param allContentNodes an object of keys and values - content type and content nodes to transform, respectively
- * @param config Flatbread config object
- */
-export const optionallyTransformContentNodes = (
-  allContentNodes: Record<string, VFile[]>,
-  config: LoadedFlatbreadConfig
-): Record<string, EntryNode[]> => {
-  if (config.transformer) {
-    const transformerMap = getTransformerExtensionMap(config.transformer);
-    // const globs = Object.entries(transformers);
-
-    /**
-     * Map through each content type,
-     * then map through each content node
-     * and transform it with the provided parser
-     *
-     * @todo if this becomes a performance bottleneck, consider overloading the source plugin API to accept a transform function so we can avoid mapping through the content nodes twice
-     * */
-
-    return map(allContentNodes, (node: VFile) => {
-      const transformer = transformerMap.get(node.extname ?? '');
-      if (!transformer?.parse) {
-        throw new Error(`no transformer found for ${node.path}`);
-      }
-      return withSourceContext(transformer.parse(node), node);
-    });
-  }
-
-  return allContentNodes as unknown as Record<string, EntryNode[]>;
-};
-
-function withSourceContext(entry: EntryNode, sourceNode: VFile): EntryNode {
-  return {
-    ...entry,
-    _path: sourceNode.path,
-    _filename: sourceNode.basename,
-  };
-}