diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2fe5b206..b94d3f0fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,14 +2,13 @@ name: Build on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] - types: [ opened, synchronize, reopened ] + branches: ["main"] + types: [opened, synchronize, reopened] jobs: build: - runs-on: ubuntu-latest strategy: @@ -17,14 +16,16 @@ jobs: node-version: [20.x, 22.x] steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - - run: npm install - - name: Run eslint - run: npm run lint - - name: Run vite build - run: npm run build + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: "npm" + - run: npm install + - name: Run tests + run: npm test + - name: Run eslint + run: npm run lint + - name: Run vite build + run: npm run build diff --git a/package.json b/package.json index 848104764..4ddf6e2b6 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "dev": "vite", "build": "vite build", "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", + "test": "node --test", "preview": "vite preview" }, "dependencies": { diff --git a/src/components/EditorHeader/Modal/Modal.jsx b/src/components/EditorHeader/Modal/Modal.jsx index 149549d3a..3ffcae337 100644 --- a/src/components/EditorHeader/Modal/Modal.jsx +++ b/src/components/EditorHeader/Modal/Modal.jsx @@ -27,6 +27,7 @@ import { import { isRtl } from "../../../i18n/utils/rtl"; import { useExtensions } from "../../../context/ExtensionsContext"; import { importSQL } from "../../../utils/importSQL"; +import { parsePostgresSQL } from "../../../utils/importSQL/postgresImplicitReferences"; import { allowedTypesFor, normalizeAiDiagram, @@ -130,10 +131,10 @@ export default function Modal({ ast = oracleParser.parse(importSource.src); } else { const parser = new Parser(); - - ast = parser.astify(importSource.src, { - database: targetDatabase, - }); + ast = + targetDatabase === DB.POSTGRES + ? parsePostgresSQL(parser, importSource.src, targetDatabase) + : parser.astify(importSource.src, { database: targetDatabase }); } } catch (error) { const message = error.location @@ -200,7 +201,10 @@ export default function Modal({ if (!result) return; - const { diagram, warnings } = normalizeAiDiagram(result.diagram, database); + const { diagram, warnings } = normalizeAiDiagram( + result.diagram, + database, + ); const allWarnings = [...(result.warnings ?? []), ...warnings]; applyImportedDiagram(diagram); diff --git a/src/utils/importSQL/postgres.js b/src/utils/importSQL/postgres.js index 397d60800..c8a143d7c 100644 --- a/src/utils/importSQL/postgres.js +++ b/src/utils/importSQL/postgres.js @@ -2,6 +2,7 @@ import { nanoid } from "nanoid"; import { Cardinality, Constraint, DB } from "../../data/constants"; import { dbToTypes } from "../../data/datatypes"; import { buildSQLFromAST } from "./shared"; +import { postgresReferenceFieldNames } from "./postgresImplicitReferences"; const affinity = { [DB.POSTGRES]: new Proxy( @@ -24,6 +25,7 @@ export function fromPostgres(ast, diagramDb = DB.GENERIC) { const relationships = []; const types = []; const enums = []; + const primaryKeys = new Map(); const parseSingleStatement = (e) => { if (e.type === "create") { @@ -134,14 +136,17 @@ export function fromPostgres(ast, diagramDb = DB.GENERIC) { (c) => c.column.expr.value, ); const endTableName = d.reference_definition.table[0].table; - const endFieldNames = d.reference_definition.definition.map( - (c) => c.column.expr.value, - ); - const startFieldName = startFieldNames[0]; - const endTable = tables.find((t) => t.name === endTableName); if (!endTable) return; + const endFieldNames = postgresReferenceFieldNames( + d.reference_definition, + endTable, + startFieldNames.length, + primaryKeys.get(endTableName), + ); + const startFieldName = startFieldNames[0]; + const fieldPairs = []; for (let i = 0; i < startFieldNames.length; i++) { const sf = table.fields.find( @@ -214,8 +219,16 @@ export function fromPostgres(ast, diagramDb = DB.GENERIC) { const startTableName = table.name; const startFieldName = field.name; const endTableName = d.reference_definition.table[0].table; - const endFieldName = - d.reference_definition.definition[0].column.expr.value; + const endTable = tables.find((t) => t.name === endTableName); + if (!endTable) return; + + const [endFieldName] = postgresReferenceFieldNames( + d.reference_definition, + endTable, + 1, + primaryKeys.get(endTableName), + ); + if (!endFieldName) return; let updateConstraint = Constraint.NONE; let deleteConstraint = Constraint.NONE; d.reference_definition.on_action.forEach((c) => { @@ -232,9 +245,6 @@ export function fromPostgres(ast, diagramDb = DB.GENERIC) { } }); - const endTable = tables.find((t) => t.name === endTableName); - if (!endTable) return; - const endField = endTable.fields.find( (f) => f.name === endFieldName, ); @@ -263,6 +273,16 @@ export function fromPostgres(ast, diagramDb = DB.GENERIC) { relationships.push(relationship); } }); + const primaryKey = e.create_definitions.find( + (definition) => definition.constraint_type === "primary key", + ); + primaryKeys.set( + table.name, + primaryKey?.definition.map((column) => column.column.expr.value) ?? + table.fields + .filter((tableField) => tableField.primary) + .map((tableField) => tableField.name), + ); tables.push(table); } else if (e.keyword === "index") { const index = { diff --git a/src/utils/importSQL/postgresImplicitReferences.js b/src/utils/importSQL/postgresImplicitReferences.js new file mode 100644 index 000000000..92dcd5e1d --- /dev/null +++ b/src/utils/importSQL/postgresImplicitReferences.js @@ -0,0 +1,303 @@ +const IMPLICIT_REFERENCE_PREFIX = "__drawdb_implicit_reference__"; + +const isIdentifierStart = (char) => /[A-Za-z_\u0080-\uFFFF]/u.test(char); +const isIdentifierPart = (char) => /[A-Za-z0-9_$\u0080-\uFFFF]/u.test(char); +const isKeyword = (token, keyword) => token?.lower === keyword; + +function skipQuoted(sql, start, quote, backslashEscapes = false) { + let index = start + 1; + while (index < sql.length) { + if (sql[index] === quote) { + if (sql[index + 1] === quote) { + index += 2; + continue; + } + return index + 1; + } + index += backslashEscapes && sql[index] === "\\" ? 2 : 1; + } + return sql.length; +} + +function skipBlockComment(sql, start) { + let index = start + 2; + let depth = 1; + while (index < sql.length && depth > 0) { + if (sql.startsWith("/*", index)) { + depth += 1; + index += 2; + } else if (sql.startsWith("*/", index)) { + depth -= 1; + index += 2; + } else { + index += 1; + } + } + return index; +} + +function tokenize(sql) { + const tokens = []; + let index = 0; + while (index < sql.length) { + const char = sql[index]; + if (/\s/u.test(char)) { + index += 1; + } else if (sql.startsWith("--", index)) { + const newline = sql.indexOf("\n", index + 2); + index = newline === -1 ? sql.length : newline + 1; + } else if (sql.startsWith("/*", index)) { + index = skipBlockComment(sql, index); + } else if ((char === "E" || char === "e") && sql[index + 1] === "'") { + const end = skipQuoted(sql, index + 1, "'", true); + tokens.push({ type: "literal", start: index, end }); + index = end; + } else if (char === "'") { + const end = skipQuoted(sql, index, char); + tokens.push({ type: "literal", start: index, end }); + index = end; + } else if (char === '"') { + const end = skipQuoted(sql, index, char); + tokens.push({ type: "identifier", start: index, end }); + index = end; + } else if (char === "$") { + const delimiter = sql + .slice(index) + .match(/^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/u)?.[0]; + if (!delimiter) { + tokens.push({ value: char, start: index, end: index + 1 }); + index += 1; + } else { + const end = sql.indexOf(delimiter, index + delimiter.length); + const quoteEnd = end === -1 ? sql.length : end + delimiter.length; + tokens.push({ type: "literal", start: index, end: quoteEnd }); + index = quoteEnd; + } + } else if (isIdentifierStart(char)) { + const start = index; + index += 1; + while (index < sql.length && isIdentifierPart(sql[index])) index += 1; + tokens.push({ + type: "identifier", + lower: sql.slice(start, index).toLowerCase(), + start, + end: index, + }); + } else { + tokens.push({ value: char, start: index, end: index + 1 }); + index += 1; + } + } + return tokens; +} + +function statementStart(tokens, index) { + while (index > 0 && tokens[index - 1].value !== ";") index -= 1; + return index; +} + +function createTableOpen(tokens, start) { + if (!isKeyword(tokens[start], "create")) return -1; + let index = start + 1; + while ( + ["global", "local", "temp", "temporary", "unlogged"].some((keyword) => + isKeyword(tokens[index], keyword), + ) + ) { + index += 1; + } + if (!isKeyword(tokens[index], "table")) return -1; + index += 1; + if (isKeyword(tokens[index], "only")) return -1; + if (isKeyword(tokens[index], "if")) { + if ( + !isKeyword(tokens[index + 1], "not") || + !isKeyword(tokens[index + 2], "exists") + ) { + return -1; + } + index += 3; + } + if (tokens[index]?.type !== "identifier") return -1; + index += 1; + while (tokens[index]?.value === ".") index += 2; + return tokens[index]?.value === "(" ? index : -1; +} + +function isTableForeignKey(tokens, itemStart) { + const start = isKeyword(tokens[itemStart], "constraint") + ? itemStart + 2 + : itemStart; + return ( + isKeyword(tokens[start], "foreign") && isKeyword(tokens[start + 1], "key") + ); +} + +function createTableReference(tokens, referenceIndex) { + const start = statementStart(tokens, referenceIndex); + const tableOpen = createTableOpen(tokens, start); + if (tableOpen === -1) return false; + + let depth = 0; + let itemStart = tableOpen + 1; + for (let index = tableOpen; index < referenceIndex; index += 1) { + if (tokens[index].value === "(") depth += 1; + else if (tokens[index].value === ")") depth -= 1; + else if (tokens[index].value === "," && depth === 1) { + itemStart = index + 1; + } + } + if (depth !== 1 || referenceIndex === itemStart) return false; + if (isKeyword(tokens[referenceIndex - 1], "constraint")) return false; + if (isTableForeignKey(tokens, itemStart)) return true; + if ( + ["constraint", "check", "exclude", "foreign", "primary", "unique"].some( + (keyword) => isKeyword(tokens[itemStart], keyword), + ) + ) { + return false; + } + return ( + tokens[itemStart]?.type === "identifier" && + referenceIndex > itemStart + 1 && + !isKeyword(tokens[referenceIndex - 1], "default") + ); +} + +function isImplicitReferenceCandidate(tokens, index) { + const target = index + 1; + const targetKeywords = [ + "check", + "collate", + "compression", + "constraint", + "default", + "deferrable", + "generated", + "initially", + "match", + "not", + "null", + "on", + "primary", + "unique", + ]; + const trailerKeywords = ["match", "on", "deferrable", "initially"]; + const afterTarget = tokens[target + 1]; + return ( + isKeyword(tokens[index], "references") && + tokens[target]?.type === "identifier" && + !isKeyword(tokens[target], "only") && + !targetKeywords.some((keyword) => isKeyword(tokens[target], keyword)) && + (afterTarget == null || + [",", ")", ";"].includes(afterTarget.value) || + trailerKeywords.some((keyword) => isKeyword(afterTarget, keyword)) || + (isKeyword(afterTarget, "not") && + isKeyword(tokens[target + 2], "deferrable"))) + ); +} + +function hasLaterCandidate(tokens, referenceIndex) { + let depth = 1; + for (let index = referenceIndex + 2; index < tokens.length; index += 1) { + if (tokens[index].value === "(") depth += 1; + else if (tokens[index].value === ")") depth -= 1; + else if (tokens[index].value === "," && depth === 1) break; + if (depth === 0) break; + if (depth === 1 && isKeyword(tokens[index], "references")) return true; + } + return false; +} + +function implicitReferenceInsertions(sql) { + const tokens = tokenize(sql); + const insertions = []; + for (let index = 0; index < tokens.length; index += 1) { + if ( + !isImplicitReferenceCandidate(tokens, index) || + !createTableReference(tokens, index) || + hasLaterCandidate(tokens, index) + ) { + continue; + } + + const target = index + 1; + insertions.push(tokens[target].end); + index = target; + } + return insertions; +} + +function unusedMarker(sql) { + let suffix = 0; + let marker = IMPLICIT_REFERENCE_PREFIX; + while (sql.toLowerCase().includes(marker.toLowerCase())) { + marker = `${IMPLICIT_REFERENCE_PREFIX}${++suffix}`; + } + return marker; +} + +export function preparePostgresSQL(sql) { + const insertions = implicitReferenceInsertions(sql); + if (insertions.length === 0) return { sql, marker: null }; + + const marker = unusedMarker(sql); + let prepared = sql; + for (let index = insertions.length - 1; index >= 0; index -= 1) { + const position = insertions[index]; + prepared = `${prepared.slice(0, position)} ("${marker}")${prepared.slice(position)}`; + } + return { sql: prepared, marker }; +} + +const columnName = (definition) => definition?.column?.expr?.value; + +export function markImplicitPostgresReferences(ast, marker) { + if (!marker) return ast; + const visit = (value) => { + if (Array.isArray(value)) return value.forEach(visit); + if (!value || typeof value !== "object") return; + const reference = value.reference_definition; + if ( + reference?.definition?.length === 1 && + columnName(reference.definition[0]) === marker + ) { + reference.implicitPrimaryKey = true; + } + Object.values(value).forEach(visit); + }; + visit(ast); + return ast; +} + +export function parsePostgresSQL(parser, sql, database) { + try { + return parser.astify(sql, { database }); + } catch (error) { + const prepared = preparePostgresSQL(sql); + if (!prepared.marker) throw error; + try { + const ast = parser.astify(prepared.sql, { database }); + return markImplicitPostgresReferences(ast, prepared.marker); + } catch { + throw error; + } + } +} + +export function postgresReferenceFieldNames( + reference, + referencedTable, + expectedCount, + primaryFieldNames = null, +) { + if (!reference?.implicitPrimaryKey) { + return reference?.definition?.map(columnName) ?? []; + } + const names = + primaryFieldNames ?? + referencedTable.fields + .filter((field) => field.primary) + .map((field) => field.name); + return names.length === expectedCount ? names : []; +} diff --git a/test/postgres-implicit-references.test.js b/test/postgres-implicit-references.test.js new file mode 100644 index 000000000..cd02b8090 --- /dev/null +++ b/test/postgres-implicit-references.test.js @@ -0,0 +1,231 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import sqlParser from "node-sql-parser"; +import { createServer } from "vite"; +import { + parsePostgresSQL, + preparePostgresSQL, +} from "../src/utils/importSQL/postgresImplicitReferences.js"; + +const { Parser } = sqlParser; + +function parsePostgres(sql) { + return parsePostgresSQL(new Parser(), sql, "Postgresql"); +} + +test("imports omitted PostgreSQL reference columns as primary-key relationships", async () => { + const server = await createServer({ + appType: "custom", + configFile: false, + optimizeDeps: { noDiscovery: true }, + server: { middlewareMode: true }, + }); + + try { + const { fromPostgres } = await server.ssrLoadModule( + "/src/utils/importSQL/postgres.js", + ); + const diagram = fromPostgres( + parsePostgres(` + CREATE TABLE test.table1 (id UUID NOT NULL PRIMARY KEY); + CREATE TABLE test.table2 ( + id UUID NOT NULL PRIMARY KEY, + ref_id UUID REFERENCES table1 + ); + `), + "generic", + ); + const [parent, child] = diagram.tables; + assert.equal(diagram.relationships.length, 1); + assert.equal(diagram.relationships[0].startFieldId, child.fields[1].id); + assert.equal(diagram.relationships[0].endFieldId, parent.fields[0].id); + + const composite = fromPostgres( + parsePostgres(` + CREATE TABLE parent (a INT, b INT, PRIMARY KEY (b, a)); + CREATE TABLE child ( + parent_b INT, + parent_a INT, + FOREIGN KEY (parent_b, parent_a) REFERENCES parent + ); + `), + "generic", + ); + assert.deepEqual( + composite.relationships[0].fields.map(({ startFieldId, endFieldId }) => [ + composite.tables[1].fields.find((field) => field.id === startFieldId) + .name, + composite.tables[0].fields.find((field) => field.id === endFieldId) + .name, + ]), + [ + ["parent_b", "b"], + ["parent_a", "a"], + ], + ); + } finally { + await server.close(); + } +}); + +test("only rewrites omitted unqualified references in CREATE TABLE", () => { + const implicit = ` + -- REFERENCES ignored_comment + CREATE TABLE child ( + note TEXT DEFAULT 'REFERENCES ignored_string', + parent_id NUMERIC(10, 2) REFERENCES parent + ); + `; + const prepared = preparePostgresSQL(implicit); + assert.ok(prepared.marker); + assert.equal(prepared.sql.split(`("${prepared.marker}")`).length - 1, 1); + assert.doesNotThrow(() => parsePostgres(implicit)); + + const afterBackslashLiteral = String.raw`CREATE TABLE child ( + note TEXT DEFAULT '\', + parent_id INT REFERENCES parent + );`; + assert.ok(preparePostgresSQL(afterBackslashLiteral).marker); + + const escapeString = String.raw`SELECT E'escaped \' REFERENCES ignored';`; + assert.deepEqual(preparePostgresSQL(escapeString), { + sql: escapeString, + marker: null, + }); + + for (const definition of [ + "id INT NOT NULL REFERENCES parent", + "id INT UNIQUE REFERENCES parent", + "id INT DEFAULT 1 REFERENCES parent", + "id TIMESTAMP DEFAULT now() REFERENCES parent", + "id UUID DEFAULT gen_random_uuid() REFERENCES parent", + "id INT DEFAULT -1 REFERENCES parent", + "id TEXT DEFAULT '1'::TEXT REFERENCES parent", + "id INT CHECK (id > 0) REFERENCES parent", + 'id TEXT COLLATE "C" REFERENCES parent', + "id DOUBLE PRECISION REFERENCES parent", + "id TIMESTAMP WITH TIME ZONE REFERENCES parent", + "id INT REFERENCES references", + "id INT REFERENCES parent ON DELETE CASCADE", + "id INT REFERENCES parent MATCH FULL", + ]) { + const sql = `CREATE TABLE child (${definition});`; + assert.ok(preparePostgresSQL(sql).marker, definition); + assert.doesNotThrow(() => parsePostgres(sql), definition); + } + + for (const sql of [ + "CREATE TABLE references (id INT);", + "CREATE TABLE child (references INT);", + "CREATE TABLE child (id INT CHECK(references > 0));", + "SELECT references FROM audit_log;", + "/* REFERENCES block_comment */ SELECT 1;", + "SELECT $$ REFERENCES dollar_quote $$;", + "GRANT REFERENCES ON parent TO app_user;", + "CREATE TABLE child (id INT REFERENCES app.parent);", + "CREATE TABLE child (id INT REFERENCES parent(id));", + "CREATE TABLE child (id INT CONSTRAINT references REFERENCES parent(id));", + "CREATE TABLE child (id INT, CONSTRAINT references FOREIGN KEY (id) REFERENCES parent(id));", + "CREATE TABLE child (id INT CONSTRAINT references UNIQUE);", + "CREATE TABLE child (id INT, CONSTRAINT references UNIQUE (id));", + "CREATE TABLE child (id TEXT COLLATE references NOT NULL);", + "CREATE TABLE child (id TEXT COMPRESSION references NOT NULL);", + "CREATE TABLE child (id INT REFERENCES ONLY parent);", + "ALTER TABLE child ADD FOREIGN KEY (id) REFERENCES parent;", + ]) { + assert.deepEqual(preparePostgresSQL(sql), { sql, marker: null }); + } + + for (const sql of [ + "CREATE TABLE child (id INT REFERENCES parent(id));", + "CREATE TABLE child (id INT CONSTRAINT references UNIQUE);", + "CREATE TABLE child (id TEXT COLLATE references NOT NULL);", + ]) { + assert.doesNotThrow(() => parsePostgres(sql)); + } + + for (const sql of [ + "CREATE TABLE child (id INT CONSTRAINT references REFERENCES parent);", + "CREATE TABLE child (id INT, CONSTRAINT references FOREIGN KEY (id) REFERENCES parent);", + "CREATE TABLE child (id references REFERENCES parent);", + ]) { + const prepared = preparePostgresSQL(sql); + assert.ok(prepared.marker); + assert.equal(prepared.sql.split(`("${prepared.marker}")`).length - 1, 1); + } + + assert.throws(() => + parsePostgres("CREATE TABLE child (id INT REFERENCES ONLY parent);"), + ); +}); + +test("only retries PostgreSQL parsing for omitted reference columns", () => { + const validCalls = []; + const validParser = { + astify(sql, options) { + validCalls.push([sql, options]); + return { type: "valid" }; + }, + }; + assert.deepEqual( + parsePostgresSQL(validParser, "CREATE TABLE valid (id INT);", "Postgresql"), + { type: "valid" }, + ); + assert.equal(validCalls.length, 1); + + const retryCalls = []; + const retryParser = { + astify(sql) { + retryCalls.push(sql); + if (retryCalls.length === 1) throw new Error("unsupported omission"); + return { + reference_definition: { + definition: [ + { + column: { + expr: { value: "__drawdb_implicit_reference__" }, + }, + }, + ], + }, + }; + }, + }; + const ast = parsePostgresSQL( + retryParser, + "CREATE TABLE child (id INT REFERENCES parent);", + "Postgresql", + ); + assert.equal(retryCalls.length, 2); + assert.equal(ast.reference_definition.implicitPrimaryKey, true); + + const originalError = new Error("original parser error"); + const failingParser = { + astify: () => { + throw originalError; + }, + }; + assert.throws( + () => parsePostgresSQL(failingParser, "SELECT references;", "Postgresql"), + (error) => error === originalError, + ); + + const retryError = new Error("retry parser error"); + let attempts = 0; + const doubleFailingParser = { + astify() { + attempts += 1; + throw attempts === 1 ? originalError : retryError; + }, + }; + assert.throws( + () => + parsePostgresSQL( + doubleFailingParser, + "CREATE TABLE child (id INT REFERENCES parent);", + "Postgresql", + ), + (error) => error === originalError, + ); + assert.equal(attempts, 2); +});