diff --git a/docs/glossary.md b/docs/glossary.md index d3493e65..03b189ad 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -47,6 +47,13 @@ When GraphQL is your **query interface**, the **generated GraphQL schema** descr **One loaded item** in a collection: the structured result of reading a file (metadata, body, derived fields) that your app treats as a single unit. “Record” here means **a document-shaped object in memory**, not a row in a remote database. +### Record production + +The transformation of collection-grouped source files into records, with core +stamping source context (`_path`, `_filename`) last. Record production excludes +validation and path ownership, which are handled by `validateRecords` and +`classifyPath`. + ### Relation A **configured link** from entries in one collection to another (for example, `refs` in config mapping a post field to an `Author` collection). Relations express **associations between flat-file content**, not foreign keys managed by a separate database server. diff --git a/packages/core/src/export/json.ts b/packages/core/src/export/json.ts index 2117e892..7fcffaa8 100644 --- a/packages/core/src/export/json.ts +++ b/packages/core/src/export/json.ts @@ -1,14 +1,12 @@ -import { VFile } from 'vfile'; import { relative } from 'node:path'; -import { generateSchema } from '../generators/schema'; import { ConfigResult, ContentEntry, EntryNode, LoadedFlatbreadConfig, - Transformer, } from '../types'; import { normalizeIdentifier } from '../utils/ids'; +import { produceRecords, validateRecords } from '../records'; export interface JsonExportOptions { collections?: readonly string[]; @@ -26,7 +24,7 @@ export type JsonExportResult = Record; * - record IDs and configured relation fields are normalized with the same ID * semantics used by query resolvers; * - `_path` is emitted relative to `pathRoot` (default: `process.cwd()`); - * - invalid IDs/refs fail through the same validation gate as schema + * - invalid IDs/refs fail through the same record validation seam as schema * generation before export output is returned. */ export async function exportCollectionsAsJson( @@ -38,13 +36,10 @@ export async function exportCollectionsAsJson( throw new Error('Config is not defined'); } - // Reuse schema generation as the validation gate so exports cannot silently - // serialize broken ids or refs. - await generateSchema(configResult); - config.source.initialize?.(config); const rawNodes = await config.source.fetch(config.content); - const transformerByExtension = getTransformerExtensionMap(config.transformer); + const produced = produceRecords(rawNodes, config); + validateRecords(produced, config); const selected = new Set( options.collections ?? config.content.map((entry) => entry.collection) ); @@ -63,12 +58,11 @@ export async function exportCollectionsAsJson( } const result: JsonExportResult = {}; - for (const [collection, nodes] of Object.entries(rawNodes)) { + for (const [collection, nodes] of Object.entries(produced)) { if (!selected.has(collection)) continue; const contentEntry = contentByCollection.get(collection); const records = nodes - .map((node) => parseNode(node, transformerByExtension)) .map((entry) => normalizeRecord(entry, contentEntry, options.pathRoot ?? process.cwd()) ) @@ -89,34 +83,6 @@ export async function exportCollectionsAsJson( ); } -function getTransformerExtensionMap( - transformer: Transformer[] -): Map { - const transformerMap = new Map(); - transformer.forEach((nextTransformer) => { - nextTransformer.extensions.forEach((extension) => { - transformerMap.set(extension, nextTransformer); - }); - }); - return transformerMap; -} - -function parseNode( - node: VFile, - transformerByExtension: Map -): EntryNode { - const transformer = transformerByExtension.get(node.extname ?? ''); - if (!transformer?.parse) { - throw new Error(`no transformer found for ${node.path}`); - } - - return { - ...transformer.parse(node), - _path: node.path, - _filename: node.basename, - }; -} - function normalizeRecord( entry: EntryNode, contentEntry: ContentEntry | undefined, diff --git a/packages/core/src/export/tests/json.test.ts b/packages/core/src/export/tests/json.test.ts index d73edb53..badd3aea 100644 --- a/packages/core/src/export/tests/json.test.ts +++ b/packages/core/src/export/tests/json.test.ts @@ -3,6 +3,7 @@ import filesystem from '@flatbread/source-filesystem'; import markdownTransformer from '@flatbread/transformer-markdown'; import { exportCollectionsAsJson } from '../json'; import { initializeConfig } from '../../utils/initializeConfig'; +import { VFile } from 'vfile'; test('exports selected collections as stable normalized JSON', async (t) => { const config = initializeConfig({ @@ -90,3 +91,27 @@ test('reuses validation diagnostics before exporting JSON', async (t) => { t.regex(error?.message ?? '', /Author record id "123" is duplicated/); }); + +test('fetches content exactly once per export', async (t) => { + const events: string[] = []; + const config = initializeConfig({ + source: { + initialize: () => events.push('initialize'), + fetch: async () => { + events.push('fetch'); + return { Post: [new VFile({ path: 'post.md', value: 'id: post' })] }; + }, + }, + transformer: { + extensions: ['.md'], + inspect: String, + parse: () => ({ id: 'post' }), + }, + content: [{ collection: 'Post', path: 'content' }], + }); + + const result = await exportCollectionsAsJson({ config }); + t.is(events.filter((event) => event === 'fetch').length, 1); + t.deepEqual(events.slice(0, 2), ['initialize', 'fetch']); + t.is(result.Post[0].id, 'post'); +}); diff --git a/packages/core/src/generators/contentGraph.ts b/packages/core/src/generators/contentGraph.ts index 421d025e..3cffed6b 100644 --- a/packages/core/src/generators/contentGraph.ts +++ b/packages/core/src/generators/contentGraph.ts @@ -1,4 +1,4 @@ -import { extname, resolve } from 'node:path'; +import { resolve } from 'node:path'; import type { VFile } from 'vfile'; import type { ContentGraphSnapshot, @@ -8,11 +8,7 @@ import type { LoadedFlatbreadConfig, } from '../types'; import { getNodeIdentifier, normalizeIdentifier } from '../utils/ids'; -import { validateCollectionReferences } from '../utils/references'; -import { - optionallyTransformContentNodes, - validateCollectionIdentifiers, -} from './schema'; +import { classifyPath, produceRecords, validateRecords } from '../records'; const keyFor = (collection: string, id: string) => `${collection}\u0000${id}`; @@ -21,9 +17,8 @@ export async function buildContentGraph( ): Promise { config.source.initialize?.(config); const fetched = await config.source.fetch(config.content); - const transformed = optionallyTransformContentNodes(fetched, config); - const nodesByCollection = validateCollectionIdentifiers(transformed); - validateCollectionReferences(transformed, config.content); + const produced = produceRecords(fetched, config); + const nodesByCollection = validateRecords(produced, config); return makeSnapshot(config, nodesByCollection); } @@ -102,8 +97,7 @@ export async function patchContentGraph( indexed.node as ContentNode ); } - validateCollectionIdentifiers(nodesByCollection); - validateCollectionReferences(nodesByCollection, previous.config.content); + validateRecords(nodesByCollection, previous.config); return makeSnapshot(previous.config, nodesByCollection); } @@ -113,10 +107,13 @@ function transformFiles( ): Map { const grouped: Record = {}; for (const file of files) { - const collection = collectionForPath(resolve(file.path), config); - if (collection) (grouped[collection] ??= []).push(file); + const classification = classifyPath(resolve(file.path), config); + if (classification) { + file.data = { ...file.data, ...classification.captures }; + (grouped[classification.collection] ??= []).push(file); + } } - const transformed = optionallyTransformContentNodes(grouped, config); + const transformed = produceRecords(grouped, config); const result = new Map(); for (const [collection, nodes] of Object.entries(transformed)) { for (const node of nodes) { @@ -187,36 +184,3 @@ function referencesFor( } return result; } - -function collectionForPath( - path: string, - config: LoadedFlatbreadConfig -): string | undefined { - const extension = extname(path).toLowerCase(); - if ( - !(config.loaded.extensions ?? []).some( - (item) => `.${item.replace(/^\./, '')}`.toLowerCase() === extension - ) - ) { - return undefined; - } - return config.content.find((entry) => { - if (!entry.path) return false; - const root = resolve(entry.path); - if (path === root || path.startsWith(`${root}/`)) return true; - return pathPatternMatches(path, root); - })?.collection; -} - -function pathPatternMatches(path: string, pattern: string): boolean { - const escaped = pattern - .split('/') - .map((segment) => { - if (/^\[[^\]]+\]/.test(segment)) { - return '[^/]+'; - } - return segment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - }) - .join('/'); - return new RegExp(`^${escaped}(?:/|$)`).test(path); -} diff --git a/packages/core/src/generators/schema.ts b/packages/core/src/generators/schema.ts index 57ea155d..34aa9a45 100644 --- a/packages/core/src/generators/schema.ts +++ b/packages/core/src/generators/schema.ts @@ -2,7 +2,6 @@ import { schemaComposer } from 'graphql-compose'; import { composeWithJson } from 'graphql-compose-json'; import { cloneDeep, merge } from 'lodash-es'; import plur from 'plur'; -import { VFile } from 'vfile'; import { cacheSchema, checkCacheForSchema } from '../cache/cache'; import { generateArgsForAllItemQuery, @@ -16,15 +15,13 @@ import { ContentNode, EntryNode, LoadedFlatbreadConfig, - Transformer, } from '../types'; -import { map } from '../utils/map'; import { getNodeIdentifier, normalizeIdentifier, normalizeOptionalIdentifier, } from '../utils/ids'; -import { validateCollectionReferences } from '../utils/references'; +import { produceRecords, validateRecords } from '../records'; import { generateCollection } from './generateCollection'; interface RootQueries { @@ -37,7 +34,8 @@ interface ResolverPayload { } /** - * Generates a GraphQL schema from content nodes. + * Generates a GraphQL schema from content nodes. A supplied content graph is + * trusted as already validated by buildContentGraph or patchContentGraph. * * @param configResult the result of the config file processing */ @@ -61,13 +59,8 @@ export async function generateSchema( } else { config.source.initialize?.(config); const allContentNodes = await config.source.fetch(config.content); - allContentNodesJSON = optionallyTransformContentNodes( - allContentNodes, - config - ); - contentNodesByCollection = - validateCollectionIdentifiers(allContentNodesJSON); - validateCollectionReferences(allContentNodesJSON, config.content); + allContentNodesJSON = produceRecords(allContentNodes, config); + contentNodesByCollection = validateRecords(allContentNodesJSON, config); } // Content validation must run before returning a cached schema because the @@ -288,104 +281,3 @@ const fetchPreknownSchemaFragments = ( {} ); }; - -export function validateCollectionIdentifiers( - allContentNodesJSON: Record -): Record { - const errors: string[] = []; - const contentNodesByCollection: Record = {}; - - Object.entries(allContentNodesJSON).forEach(([collection, nodes]) => { - const seen = new Map(); - contentNodesByCollection[collection] = []; - - nodes.forEach((node) => { - try { - const normalizedId = getNodeIdentifier(node, collection); - const existing = seen.get(normalizedId); - - if (existing) { - errors.push( - `${collection} record id "${normalizedId}" is duplicated after normalization${sourceContext( - existing - )}${sourceContext(node)}` - ); - } else { - seen.set(normalizedId, node); - } - contentNodesByCollection[collection].push(node as ContentNode); - } catch (error) { - errors.push(error instanceof Error ? error.message : String(error)); - } - }); - }); - - if (errors.length > 0) { - errors.sort(); - throw new Error( - `Flatbread found ${errors.length} invalid record ID${ - errors.length === 1 ? '' : 's' - }:\n${errors.map((message) => `- ${message}`).join('\n')}` - ); - } - - return contentNodesByCollection; -} - -function sourceContext(node: EntryNode): string { - return typeof node._path === 'string' ? ` (${node._path})` : ''; -} - -function getTransformerExtensionMap( - transformer: Transformer[] -): Map { - const transformerMap = new Map(); - transformer.forEach((t) => { - t.extensions.forEach((extension) => { - transformerMap.set(extension, t); - }); - }); - return transformerMap; -} - -/** - * Transforms the content nodes to the expected JSON format. If no transformer is defined, the content nodes are returned as is. - * - * @param allContentNodes an object of keys and values - content type and content nodes to transform, respectively - * @param config Flatbread config object - */ -export const optionallyTransformContentNodes = ( - allContentNodes: Record, - config: LoadedFlatbreadConfig -): Record => { - if (config.transformer) { - const transformerMap = getTransformerExtensionMap(config.transformer); - // const globs = Object.entries(transformers); - - /** - * Map through each content type, - * then map through each content node - * and transform it with the provided parser - * - * @todo if this becomes a performance bottleneck, consider overloading the source plugin API to accept a transform function so we can avoid mapping through the content nodes twice - * */ - - return map(allContentNodes, (node: VFile) => { - const transformer = transformerMap.get(node.extname ?? ''); - if (!transformer?.parse) { - throw new Error(`no transformer found for ${node.path}`); - } - return withSourceContext(transformer.parse(node), node); - }); - } - - return allContentNodes as unknown as Record; -}; - -function withSourceContext(entry: EntryNode, sourceNode: VFile): EntryNode { - return { - ...entry, - _path: sourceNode.path, - _filename: sourceNode.basename, - }; -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ee549967..2e4880a1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,6 +16,12 @@ export { normalizeOptionalIdentifier, } from './utils/ids'; export { validateCollectionReferences } from './utils/references'; +export { classifyPath, produceRecords, validateRecords } from './records'; +export type { + FilesByCollection, + PathClassification, + RecordsByCollection, +} from './records'; export * from './types'; export { FlatbreadProvider } from './providers/base'; diff --git a/packages/core/src/records/classify.ts b/packages/core/src/records/classify.ts new file mode 100644 index 00000000..a4254b48 --- /dev/null +++ b/packages/core/src/records/classify.ts @@ -0,0 +1,101 @@ +import { extname, relative, resolve } from 'node:path'; +import type { LoadedFlatbreadConfig } from '../types'; +import type { PathClassification } from './index'; + +function segments(path: string): string[] { + return relative(process.cwd(), resolve(path)).split('/').filter(Boolean); +} + +function patternHasGrammar(pattern: string): boolean { + return pattern.includes('*') || /\[[^\]]+\]/.test(pattern); +} + +function matchPattern( + candidate: readonly string[], + pattern: readonly string[], + candidateIndex = 0, + patternIndex = 0, + captures: Record = {} +): Record | undefined { + if (patternIndex === pattern.length) { + return candidateIndex === candidate.length ? captures : undefined; + } + const token = pattern[patternIndex]; + if (token === '**') { + for (let index = candidateIndex; index <= candidate.length; index++) { + const result = matchPattern(candidate, pattern, index, patternIndex + 1, { + ...captures, + }); + if (result) return result; + } + return undefined; + } + if (candidateIndex >= candidate.length) return undefined; + + const capture = /^\[([^\]]+)\](.*)$/.exec(token); + if (capture) { + const suffix = capture[2]; + const value = candidate[candidateIndex]; + if (suffix && !value.endsWith(suffix)) return undefined; + return matchPattern( + candidate, + pattern, + candidateIndex + 1, + patternIndex + 1, + { + ...captures, + [capture[1]]: suffix ? value.slice(0, -suffix.length) : value, + } + ); + } + if (token === '*') { + return matchPattern( + candidate, + pattern, + candidateIndex + 1, + patternIndex + 1, + { + ...captures, + } + ); + } + if (token !== candidate[candidateIndex]) return undefined; + return matchPattern( + candidate, + pattern, + candidateIndex + 1, + patternIndex + 1, + { + ...captures, + } + ); +} + +export function classifyPath( + path: string, + config: LoadedFlatbreadConfig +): PathClassification | undefined { + const extension = extname(path).toLowerCase(); + const extensions = (config.loaded.extensions ?? []).map((value) => + `.${value.replace(/^\./, '')}`.toLowerCase() + ); + if (!extensions.includes(extension)) return undefined; + + const candidate = segments(path); + for (const entry of config.content) { + if (!entry.path) continue; + const pattern = segments(entry.path); + if (!patternHasGrammar(entry.path)) { + if ( + candidate.length >= pattern.length && + pattern.every((part, index) => part === candidate[index]) + ) { + return { collection: entry.collection, captures: {} }; + } + continue; + } + const captures = matchPattern(candidate, pattern); + if (captures) return { collection: entry.collection, captures }; + } + return undefined; +} diff --git a/packages/core/src/records/index.ts b/packages/core/src/records/index.ts new file mode 100644 index 00000000..067862cf --- /dev/null +++ b/packages/core/src/records/index.ts @@ -0,0 +1,65 @@ +import type { VFile } from 'vfile'; +import type { EntryNode } from '../types'; + +export type RecordsByCollection = Record; + +export type FilesByCollection = Readonly>; + +export type PathClassification = Readonly<{ + collection: string; + captures: Readonly>; +}>; + +/** + * Record production: transform already-classified, collection-grouped source + * files into records, stamping source context last. + * + * For every file, select the transformer registered for the file's exact + * `VFile.extname` (leading dot included; later-configured transformers win a + * duplicated extension, as today). Call its `parse`, then overwrite `_path` + * with `file.path` and `_filename` with `file.basename` — core is + * authoritative for those two fields; transformer output and `file.data` + * cannot override them. All other parser output is preserved, including + * source capture data a transformer copied from `file.data`. + * + * Preserves input collection keys and file order. Throws + * `no transformer found for ` when no matching transformer/parse + * exists. When `config.transformer` is empty, returns files cast as entries + * (current no-transform behavior). Performs NO id or reference validation and + * must not classify paths or mutate `VFile.data`. + */ +export { produceRecords } from './produce'; + +/** + * Validate a COMPLETE collection→records mapping (a full graph or a full + * proposed replacement — reference validation needs every target collection + * present). Runs identifier validation first, then reference validation + * against the same unmodified record set; never partially returns after + * either fails. Preserves the exact existing aggregate, sorted diagnostic + * text of both validators byte-for-byte (validation snapshot tests depend on + * it). Returns the records narrowed to ContentNode for resolver/index use. + */ +export { validateRecords } from './validate'; + +/** + * THE single path→Collection matcher. Stable interface — a later watch + * coordinator consumes it to classify watcher events. + * + * `path` and configured `content[].path` may be absolute or relative; both + * are resolved from `process.cwd()` before comparison. Segment matching is + * case-sensitive after that normalization; extension allowlisting is + * case-insensitive and INCLUDED here (candidate extension, lowercased, with + * leading dot, must appear in `config.loaded.extensions` under the same + * normalization) — callers do not pre-filter. Returns `undefined` for an + * excluded extension or no match; never reads disk. + * + * A plain configured directory owns every descendant file. A configured + * pattern containing `[name]`, `*`, or `**` is matched against the COMPLETE + * path (no extra trailing descendants): `[name]suffix` captures the segment + * value minus the literal `suffix` (which must match); `*` consumes one + * segment without capturing; a whole `**` segment consumes zero or more + * segments without capturing. `captures` contains only named `[name]` + * values; a repeated name is overwritten by the later segment (current + * behavior). The first matching `content` entry in configuration order wins. + */ +export { classifyPath } from './classify'; diff --git a/packages/core/src/records/produce.ts b/packages/core/src/records/produce.ts new file mode 100644 index 00000000..4399f5f2 --- /dev/null +++ b/packages/core/src/records/produce.ts @@ -0,0 +1,41 @@ +import type { VFile } from 'vfile'; +import type { EntryNode, LoadedFlatbreadConfig, Transformer } from '../types'; +import type { FilesByCollection, RecordsByCollection } from './index'; + +function transformerByExtension( + transformers: readonly Transformer[] +): Map { + const result = new Map(); + for (const transformer of transformers) { + for (const extension of transformer.extensions) { + result.set(extension, transformer); + } + } + return result; +} + +function stampSourceContext(entry: EntryNode, file: VFile): EntryNode { + return { ...entry, _path: file.path, _filename: file.basename }; +} + +export function produceRecords( + files: FilesByCollection, + config: LoadedFlatbreadConfig +): RecordsByCollection { + if (config.transformer.length === 0) { + return files as unknown as RecordsByCollection; + } + + const transformers = transformerByExtension(config.transformer); + const result: RecordsByCollection = {}; + for (const [collection, collectionFiles] of Object.entries(files)) { + result[collection] = collectionFiles.map((file) => { + const transformer = transformers.get(file.extname ?? ''); + if (!transformer?.parse) { + throw new Error(`no transformer found for ${file.path}`); + } + return stampSourceContext(transformer.parse(file), file); + }); + } + return result; +} diff --git a/packages/core/src/records/tests/classify.test.ts b/packages/core/src/records/tests/classify.test.ts new file mode 100644 index 00000000..80f85fa4 --- /dev/null +++ b/packages/core/src/records/tests/classify.test.ts @@ -0,0 +1,98 @@ +import test from 'ava'; +import { resolve } from 'node:path'; +import { classifyPath } from '../classify'; +import type { LoadedFlatbreadConfig } from '../../types'; + +function config( + content: LoadedFlatbreadConfig['content'], + extensions = ['.md'] +): LoadedFlatbreadConfig { + return { content, loaded: { extensions } } as LoadedFlatbreadConfig; +} + +test('classifies a descendant of a plain collection directory', (t) => { + t.deepEqual( + classifyPath( + 'content/posts/2026/nested/post.md', + config([{ collection: 'Post', path: 'content/posts' }]) + ), + { collection: 'Post', captures: {} } + ); +}); + +test('extracts a named capture from a file-pattern segment', (t) => { + t.deepEqual( + classifyPath( + 'content/hello.md', + config([{ collection: 'Post', path: 'content/[slug].md' }]) + ), + { collection: 'Post', captures: { slug: 'hello' } } + ); +}); + +test('extracts nested captures in path order', (t) => { + t.deepEqual( + classifyPath( + 'content/news/hello.md', + config([{ collection: 'Post', path: 'content/[category]/[slug].md' }]) + ), + { collection: 'Post', captures: { category: 'news', slug: 'hello' } } + ); + t.deepEqual( + classifyPath( + 'content/a/b.md', + config([{ collection: 'Post', path: 'content/[name]/[name].md' }]) + ), + { collection: 'Post', captures: { name: 'b' } } + ); +}); + +test('filters extensions before classifying paths', (t) => { + const content = [{ collection: 'Post', path: 'content' }]; + t.is(classifyPath('content/post.txt', config(content)), undefined); + t.deepEqual(classifyPath('content/post.MD', config(content, ['MD'])), { + collection: 'Post', + captures: {}, + }); +}); + +test('returns undefined for paths outside every configured collection', (t) => { + t.is( + classifyPath( + 'content/posts-extra/post.md', + config([{ collection: 'Post', path: 'content/posts' }]) + ), + undefined + ); + t.is( + classifyPath( + 'content/hello.md/extra', + config([{ collection: 'Post', path: 'content/[slug].md' }]) + ), + undefined + ); +}); + +test('matches globstar across zero and multiple directories', (t) => { + const content = [{ collection: 'Post', path: 'content/**/[slug].md' }]; + t.deepEqual(classifyPath('content/hello.md', config(content)), { + collection: 'Post', + captures: { slug: 'hello' }, + }); + t.deepEqual(classifyPath('content/a/b/hello.md', config(content)), { + collection: 'Post', + captures: { slug: 'hello' }, + }); + t.is(classifyPath('other/hello.md', config(content)), undefined); +}); + +test('resolves relative and absolute candidates identically', (t) => { + const content = [ + { collection: 'First', path: 'content/[slug].md' }, + { collection: 'Second', path: 'content/hello.md' }, + ]; + const relative = classifyPath('content/hello.md', config(content)); + const absolute = classifyPath(resolve('content/hello.md'), config(content)); + t.deepEqual(absolute, relative); + t.is(relative?.collection, 'First'); +}); diff --git a/packages/core/src/records/tests/produce.test.ts b/packages/core/src/records/tests/produce.test.ts new file mode 100644 index 00000000..f5576c2e --- /dev/null +++ b/packages/core/src/records/tests/produce.test.ts @@ -0,0 +1,105 @@ +import test from 'ava'; +import { VFile } from 'vfile'; +import { produceRecords } from '../produce'; +import type { LoadedFlatbreadConfig, Transformer } from '../../types'; + +function config(transformer: Transformer[]): LoadedFlatbreadConfig { + return { transformer } as LoadedFlatbreadConfig; +} + +test('routes each VFile to the transformer registered for its extension', (t) => { + const calls: string[] = []; + const markdown: Transformer = { + extensions: ['.md'], + inspect: String, + parse: (file) => { + calls.push(`md:${file.path}`); + return { id: 'md' }; + }, + }; + const yaml: Transformer = { + extensions: ['.yaml'], + inspect: String, + parse: (file) => { + calls.push(`yaml:${file.path}`); + return { id: 'yaml' }; + }, + }; + const result = produceRecords( + { + First: [new VFile({ path: 'a.md', value: '' })], + Second: [new VFile({ path: 'b.yaml', value: '' })], + }, + config([markdown, yaml]) + ); + t.deepEqual(calls, ['md:a.md', 'yaml:b.yaml']); + t.deepEqual(Object.keys(result), ['First', 'Second']); + t.deepEqual( + result.First.map((node) => node.id), + ['md'] + ); + t.deepEqual( + result.Second.map((node) => node.id), + ['yaml'] + ); +}); + +test('rejects a VFile without a matching transformer', (t) => { + const error = t.throws(() => + produceRecords( + { Missing: [new VFile({ path: 'virtual/missing.txt', value: '' })] }, + config([{ extensions: ['.md'], inspect: String }]) + ) + ); + t.is(error?.message, 'no transformer found for virtual/missing.txt'); +}); + +test('core source context overwrites transformer path and filename', (t) => { + const file = new VFile({ path: 'virtual/real.md', value: '' }); + const result = produceRecords( + { Docs: [file] }, + config([ + { + extensions: ['.md'], + inspect: String, + parse: () => ({ + _path: 'wrong', + _filename: 'wrong.md', + _slug: 'real', + }), + }, + ]) + ); + t.deepEqual(result.Docs[0], { + _path: file.path, + _filename: file.basename, + _slug: 'real', + }); +}); + +test('preserves capture data spread by the transformer', (t) => { + const file = new VFile({ path: 'virtual/hello.md', value: '' }); + file.data = { category: 'news', slug: 'hello' }; + const result = produceRecords( + { Docs: [file] }, + config([ + { + extensions: ['.md'], + inspect: String, + parse: (input) => ({ + ...input.data, + category: 'document', + id: 'hello', + }), + }, + ]) + ); + t.deepEqual(result.Docs[0], { + category: 'document', + slug: 'hello', + id: 'hello', + _path: file.path, + _filename: file.basename, + }); + t.deepEqual(file.data, { category: 'news', slug: 'hello' }); +}); diff --git a/packages/core/src/records/tests/sourceFilesystemParity.test.ts b/packages/core/src/records/tests/sourceFilesystemParity.test.ts new file mode 100644 index 00000000..e1f8ae61 --- /dev/null +++ b/packages/core/src/records/tests/sourceFilesystemParity.test.ts @@ -0,0 +1,37 @@ +import test from 'ava'; +import { resolve } from 'node:path'; +import filesystem from '@flatbread/source-filesystem'; +import type { LoadedFlatbreadConfig } from '../../types'; +import { classifyPath } from '../classify'; + +const root = 'packages/source-filesystem/src/utils/tests/fixtures/captures'; +const paths = [ + `${root}/news/hello.md`, + `${root}/tech/world.md`, + `${root}/tech/notes.txt`, + `${root}/news/missing.md`, +]; + +test('fetchPaths capture data and inclusion match classifyPath', async (t) => { + const content = [ + { collection: 'Plain', path: `${root}/news` }, + { collection: 'Capture', path: `${root}/[category]/[slug].md` }, + { collection: 'Recursive', path: `${root}/**/[slug].md` }, + ]; + const config = { + content, + loaded: { extensions: ['.md'] }, + } as unknown as LoadedFlatbreadConfig; + const plugin = filesystem(); + plugin.initialize?.(config); + const files = await plugin.fetchPaths!(paths); + const byPath = new Map(files.map((file) => [resolve(file.path), file])); + for (const path of paths) { + const expected = path.endsWith('/missing.md') + ? undefined + : classifyPath(path, config); + const actual = byPath.get(resolve(path)); + t.is(Boolean(actual), expected !== undefined, path); + if (actual && expected) t.deepEqual(actual.data, expected.captures, path); + } +}); diff --git a/packages/core/src/records/tests/validate.test.ts b/packages/core/src/records/tests/validate.test.ts new file mode 100644 index 00000000..1d7b2365 --- /dev/null +++ b/packages/core/src/records/tests/validate.test.ts @@ -0,0 +1,81 @@ +import test from 'ava'; +import { validateRecords } from '../validate'; +import type { LoadedFlatbreadConfig } from '../../types'; + +const config = ( + content: LoadedFlatbreadConfig['content'] +): LoadedFlatbreadConfig => ({ content } as LoadedFlatbreadConfig); + +test('reports duplicate normalized IDs with the existing sorted message', (t) => { + const error = t.throws(() => + validateRecords( + { + Author: [ + { id: ' 123 ', _path: 'b.md' }, + { id: 123, _path: 'a.md' }, + ], + }, + config([{ collection: 'Author' }]) + ) + ); + t.is( + error?.message, + 'Flatbread found 1 invalid record ID:\n- Author record id "123" is duplicated after normalization (b.md) (a.md)' + ); +}); + +test('reports invalid IDs with the existing aggregate text', (t) => { + const error = t.throws(() => + validateRecords( + { Author: [{ id: false }, { id: '' }, { id: 'valid' }] }, + config([{ collection: 'Author' }]) + ) + ); + t.is( + error?.message, + 'Flatbread found 2 invalid record IDs:\n- Author record id must be a non-empty string or finite number identifier.\n- Author record id must be a non-empty string or finite number identifier.' + ); +}); + +test('delegates reference validation without changing diagnostics', (t) => { + const error = t.throws(() => + validateRecords( + { + Author: [{ id: 'known' }], + Post: [ + { + id: 'post', + author: 'missing', + authors: ['missing', false], + ghost: 'nobody', + }, + ], + }, + config([ + { + collection: 'Post', + refs: { author: 'Author', authors: 'Author', ghost: 'Missing' }, + }, + ]) + ) + ); + t.is( + error?.message, + 'Flatbread found 4 broken references:\n' + + '- Post.author (in record id "post") references "missing" but no record with that id exists in collection Author\n' + + '- Post.authors[0] (in record id "post") references "missing" but no record with that id exists in collection Author\n' + + '- Post.authors[1] (in record id "post") has an invalid reference value for collection Author: reference value must be a non-empty string or finite number identifier.\n' + + '- Post.ghost (in record id "post") declares a reference to collection Missing, but no such collection is configured' + ); +}); + +test('checks identifiers before references', (t) => { + const error = t.throws(() => + validateRecords( + { Post: [{ id: '', author: 'missing' }] }, + config([{ collection: 'Post', refs: { author: 'Author' } }]) + ) + ); + t.regex(error?.message ?? '', /^Flatbread found 1 invalid record ID/); + t.false((error?.message ?? '').includes('broken reference')); +}); diff --git a/packages/core/src/records/validate.ts b/packages/core/src/records/validate.ts new file mode 100644 index 00000000..4dec5c76 --- /dev/null +++ b/packages/core/src/records/validate.ts @@ -0,0 +1,57 @@ +import type { ContentNode, EntryNode, LoadedFlatbreadConfig } from '../types'; +import { getNodeIdentifier } from '../utils/ids'; +import { validateCollectionReferences } from '../utils/references'; +import type { RecordsByCollection } from './index'; + +function sourceContext(node: EntryNode): string { + return typeof node._path === 'string' ? ` (${node._path})` : ''; +} + +function validateIdentifiers( + records: RecordsByCollection +): Record { + const errors: string[] = []; + const contentNodesByCollection: Record = {}; + + Object.entries(records).forEach(([collection, nodes]) => { + const seen = new Map(); + contentNodesByCollection[collection] = []; + nodes.forEach((node) => { + try { + const normalizedId = getNodeIdentifier(node, collection); + const existing = seen.get(normalizedId); + if (existing) { + errors.push( + `${collection} record id "${normalizedId}" is duplicated after normalization${sourceContext( + existing + )}${sourceContext(node)}` + ); + } else { + seen.set(normalizedId, node); + } + contentNodesByCollection[collection].push(node as ContentNode); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + }); + }); + + if (errors.length > 0) { + errors.sort(); + throw new Error( + `Flatbread found ${errors.length} invalid record ID${ + errors.length === 1 ? '' : 's' + }:\n${errors.map((message) => `- ${message}`).join('\n')}` + ); + } + return contentNodesByCollection; +} + +export function validateRecords( + records: RecordsByCollection, + config: LoadedFlatbreadConfig +): Record { + const contentNodesByCollection = validateIdentifiers(records); + validateCollectionReferences(records, config.content); + return contentNodesByCollection; +} diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index adc4cc34..f78ab1e8 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -76,12 +76,16 @@ export interface ConfigResult { */ export interface Transformer { /** - * Parse a given source file into its contained data fields and an unnormalized representation of the content. + * Parse a source file into its data. Transformers that want source path + * captures must spread `input.data`; document data wins conflicts when it is + * spread after captures. Core overwrites `_path` and `_filename` after parse. * @param input Node to transform */ parse?: (input: VFile) => EntryNode; + /** GraphQL-schema-construction-only fragments; never used by record production or validation. */ preknownSchemaFragments?: () => Record; inspect: (input: EntryNode) => string; + /** Parser-routing keys match exact `VFile.extname`; later transformers win duplicates. */ extensions: string[]; } @@ -109,10 +113,27 @@ export interface ContentEntry< export interface Source { initialize?: (flatbreadConfig: LoadedFlatbreadConfig) => void; fetchByType?: (path: string) => Promise; + /** Fetch grouped files; fetchPaths fetches flat files. Capture patterns place values in VFile.data. */ fetchPaths?: (paths: readonly string[]) => Promise; fetch: (allContentTypes: Content) => Promise>; } +/** + * Source-context fields on a produced record. Transformers may stamp `_path` + * and `_filename` provisionally, but record production overwrites both from + * the source VFile after parse returns — core is authoritative. `_slug` is + * transformer-derived and is never stamped by core. + */ +export type SourceContextFields = { + _path?: string; + _filename?: string; + _slug?: string; +}; + +/** + * A valid snapshot is built from a complete produceRecords→validateRecords + * pass; generateSchema trusts a supplied graph as already validated. + */ export interface ContentGraphSnapshot { readonly config: LoadedFlatbreadConfig; readonly nodesByCollection: Readonly>; diff --git a/packages/core/src/utils/references.ts b/packages/core/src/utils/references.ts index 449cc07a..d22f169b 100644 --- a/packages/core/src/utils/references.ts +++ b/packages/core/src/utils/references.ts @@ -172,7 +172,7 @@ function collectIdsByCollection( try { ids.add(getNodeIdentifier(node, collection)); } catch { - // Invalid ids are surfaced by validateCollectionIdentifiers; skip + // Invalid ids are surfaced by validateRecords; skip // them here so the missing-ref pass can still report what it can. } } diff --git a/packages/source-filesystem/src/index.ts b/packages/source-filesystem/src/index.ts index 6ebbcfff..965bad2b 100644 --- a/packages/source-filesystem/src/index.ts +++ b/packages/source-filesystem/src/index.ts @@ -1,5 +1,4 @@ import { defaultsDeep } from 'lodash-es'; -import { resolve, extname } from 'node:path'; import { read } from 'to-vfile'; import type { LoadedFlatbreadConfig, SourcePlugin } from '@flatbread/core'; @@ -9,7 +8,8 @@ import type { InitializedSourceFilesystemConfig, sourceFilesystemConfig, } from './types'; -import gatherFileNodes, { getCaptureData } from './utils/gatherFileNodes'; +import gatherFileNodes from './utils/gatherFileNodes'; +import { matchPath } from './utils/matchPath'; /** * Get nodes (files) from the directory @@ -68,38 +68,18 @@ async function getNodesFromPaths( content: LoadedFlatbreadConfig['content'], config: InitializedSourceFilesystemConfig ): Promise { - const extensions = config.extensions.map((extension) => - extension.startsWith('.') - ? extension.toLowerCase() - : `.${extension.toLowerCase()}` - ); const files = await Promise.all( - paths - .filter((path) => extensions.includes(extname(path).toLowerCase())) - .map(async (path) => { - try { - const file = await read(path); - const entry = content.find((candidate) => { - if (!candidate.path) return false; - const patternParts = resolve(candidate.path) - .split('/') - .filter(Boolean); - const pathParts = resolve(path).split('/').filter(Boolean); - if (patternParts.length !== pathParts.length) return false; - return patternParts.every((part, index) => { - if (/^\[[^\]]+\].*$/.test(part) || part.includes('*')) - return true; - return part === pathParts[index]; - }); - }); - if (entry?.path) { - file.data = getCaptureData(resolve(path), resolve(entry.path)); - } - return file; - } catch { - return undefined; - } - }) + paths.map(async (path) => { + try { + const file = await read(path); + const match = matchPath(path, content, config.extensions); + if (!match) return undefined; + file.data = match.captures; + return file; + } catch { + return undefined; + } + }) ); return files.filter((file): file is VFile => Boolean(file)); } diff --git a/packages/source-filesystem/src/utils/gatherFileNodes.ts b/packages/source-filesystem/src/utils/gatherFileNodes.ts index f7c90092..8f880636 100644 --- a/packages/source-filesystem/src/utils/gatherFileNodes.ts +++ b/packages/source-filesystem/src/utils/gatherFileNodes.ts @@ -18,28 +18,6 @@ function getSegmentData(node: { name: string }, segment: { remove: number }) { return node.name.slice(0, node.name.length - segment.remove); } -export function getCaptureData( - path: string, - pattern: string -): Record { - const patternParts = pattern.split('/').filter(Boolean); - const pathParts = path.split('/').filter(Boolean); - if (patternParts.length !== pathParts.length) return {}; - const data: Record = {}; - for (const [index, part] of patternParts.entries()) { - const value = pathParts[index]; - const capture = /^\[([^\]]+)\](.*)$/.exec(part); - if (capture) { - const suffix = capture[2]; - if (suffix && !value.endsWith(suffix)) return {}; - data[capture[1]] = suffix ? value.slice(0, -suffix.length) : value; - } else if (part !== value && !part.includes('*')) { - return {}; - } - } - return data; -} - function processFile(segment: Segment, node: FileNode) { return (file: FileNode) => { if (!segment) return file; diff --git a/packages/source-filesystem/src/utils/matchPath.ts b/packages/source-filesystem/src/utils/matchPath.ts new file mode 100644 index 00000000..72fba372 --- /dev/null +++ b/packages/source-filesystem/src/utils/matchPath.ts @@ -0,0 +1,82 @@ +import { extname, relative, resolve } from 'node:path'; +import type { ContentEntry } from '@flatbread/core'; + +type Match = { collection: string; captures: Record }; + +function pathSegments(path: string): string[] { + return relative(process.cwd(), resolve(path)).split('/').filter(Boolean); +} + +function match( + candidate: readonly string[], + pattern: readonly string[], + candidateIndex = 0, + patternIndex = 0, + captures: Record = {} +): Record | undefined { + if (patternIndex === pattern.length) { + return candidateIndex === candidate.length ? captures : undefined; + } + const token = pattern[patternIndex]; + if (token === '**') { + for (let index = candidateIndex; index <= candidate.length; index++) { + const found = match(candidate, pattern, index, patternIndex + 1, { + ...captures, + }); + if (found) return found; + } + return undefined; + } + if (candidateIndex >= candidate.length) return undefined; + const capture = /^\[([^\]]+)\](.*)$/.exec(token); + if (capture) { + const value = candidate[candidateIndex]; + const suffix = capture[2]; + if (suffix && !value.endsWith(suffix)) return undefined; + return match(candidate, pattern, candidateIndex + 1, patternIndex + 1, { + ...captures, + [capture[1]]: suffix ? value.slice(0, -suffix.length) : value, + }); + } + if (token === '*') { + return match(candidate, pattern, candidateIndex + 1, patternIndex + 1, { + ...captures, + }); + } + if (token !== candidate[candidateIndex]) return undefined; + return match(candidate, pattern, candidateIndex + 1, patternIndex + 1, { + ...captures, + }); +} + +export function matchPath( + path: string, + content: readonly ContentEntry[], + extensions: readonly string[] +): Match | undefined { + const extension = extname(path).toLowerCase(); + const allowed = extensions.map((value) => + `.${value.replace(/^\./, '')}`.toLowerCase() + ); + if (!allowed.includes(extension)) return undefined; + + const candidate = pathSegments(path); + for (const entry of content) { + if (!entry.path) continue; + const pattern = pathSegments(entry.path); + const hasGrammar = + entry.path.includes('*') || /\[[^\]]+\]/.test(entry.path); + if (!hasGrammar) { + if ( + candidate.length >= pattern.length && + pattern.every((part, index) => part === candidate[index]) + ) { + return { collection: entry.collection, captures: {} }; + } + continue; + } + const captures = match(candidate, pattern); + if (captures) return { collection: entry.collection, captures }; + } + return undefined; +} diff --git a/packages/source-filesystem/src/utils/tests/fetchPaths.test.ts b/packages/source-filesystem/src/utils/tests/fetchPaths.test.ts index 95c0d3ce..82213da6 100644 --- a/packages/source-filesystem/src/utils/tests/fetchPaths.test.ts +++ b/packages/source-filesystem/src/utils/tests/fetchPaths.test.ts @@ -85,3 +85,26 @@ test('fetchPaths produces the same capture metadata as fetch for a [category]/[s ); t.deepEqual(pathByPath.get(resolve(WORLD_PATH))!.data, fetchedWorld!.data); }); + +test('fetchPaths supports globstar capture patterns', async (t) => { + const plugin = source(); + plugin.initialize?.({ + content: [ + { + path: 'packages/source-filesystem/src/utils/tests/fixtures/captures/**/[slug].md', + collection: 'Recursive', + }, + ], + loaded: { extensions: ['.md'] }, + } as unknown as LoadedFlatbreadConfig); + + const files = await plugin.fetchPaths!([HELLO_PATH, WORLD_PATH]); + t.is(files.length, 2); + t.deepEqual( + Object.fromEntries(files.map((file) => [resolve(file.path), file.data])), + { + [resolve(HELLO_PATH)]: { slug: 'hello' }, + [resolve(WORLD_PATH)]: { slug: 'world' }, + } + ); +}); diff --git a/packages/transformer-markdown/src/index.ts b/packages/transformer-markdown/src/index.ts index 54d5a01a..e0324219 100644 --- a/packages/transformer-markdown/src/index.ts +++ b/packages/transformer-markdown/src/index.ts @@ -10,6 +10,9 @@ export * from './types'; /** * Transforms a markdown file (content node) to JSON containing any frontmatter data or content. + * `_filename` and `_path` are provisional source context; core overwrites them + * after parsing. `_slug` is transformer-derived. Input captures are spread + * before document data, so document data wins conflicts. * * @param {VFile} input - A VFile object representing a content node. * @param {MarkdownTransformerConfig} config - A configuration object. diff --git a/packages/transformer-yaml/src/index.ts b/packages/transformer-yaml/src/index.ts index f5b36ed3..28dfb8a9 100644 --- a/packages/transformer-yaml/src/index.ts +++ b/packages/transformer-yaml/src/index.ts @@ -6,6 +6,9 @@ import type { VFile } from 'vfile'; /** * Transforms a yaml file (content node) to JSON. + * `_filename` and `_path` are provisional source context; core overwrites them + * after parsing. `_slug` is transformer-derived. Input captures are spread + * before document data, so document data wins conflicts. * * @param {VFile} input - A VFile object representing a content node. */