feat(sql): upgrade JSqlParser to 5.4 with token-based syntax reasons

docs/postgresql.md

@@ -251,7 +251,7 @@ query, its source, its header line, and the offending column and recorded type:
 
 ```text
 sqlcj: Invalid schema source /home/dev/project/schema.sql: Unsupported schema statement: Alter
-sqlcj: Invalid schema source /home/dev/project/schema.sql: Encountered unexpected token: ";" <ST_SEMICOLON> at line 4, column 1
+sqlcj: Invalid schema source /home/dev/project/schema.sql: Encountered unexpected token: ";" at line 4, column 1
 sqlcj: Invalid query 'ListTags' in /home/dev/project/queries.sql at line 5: Column 'tags' has unsupported type VARCHAR[]
 ```
 

pom.xml

@@ -25,7 +25,7 @@
         <spotless.java.config>${project.basedir}/config/eclipse-java-formatter.xml</spotless.java.config>
         <picocli.version>4.7.7</picocli.version>
         <jackson.version>3.2.1</jackson.version>
-        <jsqlparser.version>5.3</jsqlparser.version>
+        <jsqlparser.version>5.4</jsqlparser.version>
         <junit.version>6.1.2</junit.version>
         <h2.version>2.4.240</h2.version>
         <testcontainers.version>1.21.4</testcontainers.version>

sqlcj-cli/src/main/java/dev/sqlcj/sql/SqlParameterCompiler.java

@@ -2,7 +2,6 @@
 
 import net.sf.jsqlparser.parser.CCJSqlParserConstants;
 import net.sf.jsqlparser.parser.Node;
-import net.sf.jsqlparser.parser.SimpleNode;
 import net.sf.jsqlparser.parser.Token;
 
 import java.util.ArrayList;
@@ -47,7 +46,7 @@ final class SqlParameterCompiler {
     private static final int SOURCE_OFFSET = 1;
 
     SqlParameters compile(String sql, Node astRoot) {
-        if (!(astRoot instanceof SimpleNode node)) {
+        if (!(astRoot instanceof Node node)) {
             throw new SqlParseException("SQL parse tree is unavailable.");
         }
 

sqlcj-cli/src/main/java/dev/sqlcj/sql/SqlParseReason.java

@@ -1,17 +1,20 @@
 package dev.sqlcj.sql;
 
+import net.sf.jsqlparser.parser.CCJSqlParserConstants;
 import net.sf.jsqlparser.parser.ParseException;
 import net.sf.jsqlparser.parser.Token;
 
 /**
  * The corrective fact of a SQL syntax failure, such as {@code Encountered
- * unexpected token: ";" <ST_SEMICOLON>}.
+ * unexpected token: ";"}.
  *
  * <p>The SQL parser reports a syntax failure through a chain of wrapping
  * exceptions whose outer messages repeat the class name of the failure they
- * wrap, so the fact is taken from the {@link ParseException} that chain
- * carries, or from its innermost cause when it carries none, and never names an
- * exception class.
+ * wrap, and whose {@link ParseException} states the unexpected token with
+ * parser-internal token kinds and lexical states. The fact is therefore
+ * composed from the unexpected token that chain carries, or taken from the
+ * innermost cause when it carries no {@link ParseException}, and never names an
+ * exception class or a parser internal.
  */
 public final class SqlParseReason {
 
@@ -32,20 +35,41 @@ public static String of(Throwable failure, boolean withLocation) {
             return firstLine(innermostCause(failure));
         }
 
-        String reason = firstLine(parseFailure);
-
         Token token = parseFailure.currentToken == null
             ? null
             : parseFailure.currentToken.next;
 
-        if (!withLocation || token == null) {
+        if (token == null) {
+            return firstLine(parseFailure);
+        }
+
+        String reason = unexpected(token);
+
+        if (!withLocation) {
             return reason;
         }
 
         return "%s at line %d, column %d"
             .formatted(reason, token.beginLine, token.beginColumn);
     }
 
+    /**
+     * States the token the parser did not expect. The end of input has no image
+     * to quote, and a token image that spans lines is quoted with its line
+     * breaks escaped so the reason stays one line.
+     */
+    private static String unexpected(Token token) {
+        if (token.kind == CCJSqlParserConstants.EOF) {
+            return "Encountered unexpected end of input";
+        }
+
+        return "Encountered unexpected token: \"%s\"".formatted(
+            token.image
+                .replace("\r", "\\r")
+                .replace("\n", "\\n")
+        );
+    }
+
     /** The parse failure the exception chain carries, or {@code null}. */
     private static ParseException parseFailure(Throwable failure) {
         for (Throwable cause = failure; cause != null; cause = cause.getCause()) {

sqlcj-cli/src/test/java/dev/sqlcj/cli/command/GenerateCommandTest.java

@@ -205,7 +205,7 @@ name VARCHAR(255)
             List.of(
                 "sqlcj: Invalid schema source %s: "
                     .formatted(workingDirectory.resolve("schema.sql"))
-                    + "Encountered unexpected token: \";\" <ST_SEMICOLON> "
+                    + "Encountered unexpected token: \";\" "
                     + "at line 4, column 1"
             ),
             result.error().lines().toList()
@@ -248,7 +248,7 @@ void shouldFailWithConciseDiagnosticForQuerySyntaxError() throws Exception {
             List.of(
                 "sqlcj: Invalid query 'GetUser' in %s at line 1: "
                     .formatted(workingDirectory.resolve("queries.sql"))
-                    + "Encountered unexpected token: \"AND\" \"AND\""
+                    + "Encountered unexpected token: \"AND\""
             ),
             result.error().lines().toList()
         );

sqlcj-cli/src/test/java/dev/sqlcj/compiler/SqlcjCompilerIntegrationTest.java

@@ -2310,7 +2310,7 @@ CREATE TABLE orders (
 
         assertEquals(
             "Invalid schema source %s: ".formatted(broken)
-                + "Encountered unexpected token: \";\" <ST_SEMICOLON> "
+                + "Encountered unexpected token: \";\" "
                 + "at line 4, column 1",
             exception.getMessage()
         );

sqlcj-cli/src/test/java/dev/sqlcj/schema/parser/DefaultSchemaParserTest.java

@@ -553,7 +553,30 @@ name VARCHAR(255)
         );
 
         assertEquals(
-            "Encountered unexpected token: \";\" <ST_SEMICOLON> at line 4, column 1",
+            "Encountered unexpected token: \";\" at line 4, column 1",
+            exception.getMessage()
+        );
+    }
+
+    /**
+     * A syntax failure at the end of input has no token image to quote, so the
+     * reason states the end of input and where the input ended.
+     */
+    @Test
+    void shouldReportSyntaxFailureAtEndOfInputWithItsLocation() {
+        String sql = """
+            CREATE TABLE users (
+             id BIGINT NOT NULL,
+             name VARCHAR(255)
+            """;
+
+        SchemaParseException exception = assertThrows(
+            SchemaParseException.class,
+            () -> parser.parse(sql)
+        );
+
+        assertEquals(
+            "Encountered unexpected end of input at line 3, column 19",
             exception.getMessage()
         );
     }
@@ -578,7 +601,7 @@ CREATE TABLE users (
 
         assertEquals(
             "Lexical error at line 4, column 0."
-                + "  Encountered: <EOF> after prefix \"\\'x);\\n\"",
+                + "  Encountered: <EOF> after: \"\\'x);\\n\"",
             exception.getMessage()
         );
         assertFalse(exception.getMessage().contains("net.sf.jsqlparser"));

sqlcj-cli/src/test/java/dev/sqlcj/sql/SqlParameterCompilerTest.java

@@ -1,7 +1,7 @@
 package dev.sqlcj.sql;
 
 import net.sf.jsqlparser.parser.CCJSqlParserConstants;
-import net.sf.jsqlparser.parser.SimpleNode;
+import net.sf.jsqlparser.parser.Node;
 import net.sf.jsqlparser.parser.Token;
 import org.junit.jupiter.api.Test;
 
@@ -107,7 +107,7 @@ private SqlParameters compile(String sql, Token... tokens) {
             tokens[index].next = tokens[index + 1];
         }
 
-        SimpleNode node = new SimpleNode(0);
+        Node node = new Node(0);
         node.jjtSetFirstToken(tokens[0]);
         node.jjtSetLastToken(tokens[tokens.length - 1]);
 

sqlcj-cli/src/test/java/dev/sqlcj/sql/SqlParserTest.java

@@ -128,17 +128,25 @@ void shouldNotReportAnonymousParameterForProtectedText() {
         );
     }
 
+    /** Dollar-quoted text is a literal, so a {@code $N} inside it is text. */
     @Test
-    void shouldRejectDollarQuotedText() {
-        assertThrows(
-            SqlParseException.class,
-            () -> parser.parse("SELECT $$ $1 dollar quoted $$ FROM users WHERE id = $1")
+    void shouldPreserveParameterTextInsideDollarQuotedText() {
+        ParsedSql parsedSql = parser.parse(
+            "SELECT $$ $1 dollar quoted $$ FROM users WHERE id = $1"
         );
+
+        assertEquals(
+            "SELECT $$ $1 dollar quoted $$ FROM users WHERE id = ?",
+            parsedSql.parameters().executableSql()
+        );
+
+        assertEquals(List.of(1), parsedSql.parameters().indexes());
     }
 
     /**
      * The compiler names the failing query and its header line, so the reason
-     * carries the parser's own wording without an exception class name.
+     * states the unexpected token alone, without a location and without an
+     * exception class name.
      */
     @Test
     void shouldReportSyntaxFailureWithoutExceptionClassNames() {
@@ -152,7 +160,32 @@ void shouldReportSyntaxFailureWithoutExceptionClassNames() {
         );
 
         assertEquals(
-            "Encountered unexpected token: \"AND\" \"AND\"",
+            "Encountered unexpected token: \"AND\"",
+            exception.getMessage()
+        );
+    }
+
+    /** A syntax failure at the end of input has no token image to quote. */
+    @Test
+    void shouldReportSyntaxFailureAtEndOfInput() {
+        SqlParseException exception = assertThrows(
+            SqlParseException.class,
+            () -> parser.parse("SELECT id FROM users WHERE id =")
+        );
+
+        assertEquals("Encountered unexpected end of input", exception.getMessage());
+    }
+
+    /** A reason stays one line, so a token image that spans lines is escaped. */
+    @Test
+    void shouldEscapeLineBreaksOfTheUnexpectedToken() {
+        SqlParseException exception = assertThrows(
+            SqlParseException.class,
+            () -> parser.parse("SELECT id FROM users WHERE id = 'x' 'a\nb'")
+        );
+
+        assertEquals(
+            "Encountered unexpected token: \"'a\\nb'\"",
             exception.getMessage()
         );
     }
@@ -171,21 +204,21 @@ void shouldReportLexicalFailureWithoutExceptionClassNames() {
 
         assertEquals(
             "Lexical error at line 1, column 46."
-                + "  Encountered: <EOF> after prefix \"\\'abc;\"",
+                + "  Encountered: <EOF> after: \"\\'abc;\"",
             exception.getMessage()
         );
         assertFalse(exception.getMessage().contains("net.sf.jsqlparser"));
     }
 
-    /** A block comment ends at its first delimiter and does not nest. */
+    /** A block comment nests as in PostgreSQL and ends at its last delimiter. */
     @Test
-    void shouldPreserveParameterTextInsideBlockCommentWithNestedDelimiter() {
+    void shouldPreserveParameterTextInsideNestedBlockComment() {
         ParsedSql parsedSql = parser.parse(
-            "SELECT id FROM users /* outer /* $8 inner */ WHERE id = $1"
+            "SELECT id FROM users /* outer /* $8 inner */ $9 still comment */ WHERE id = $1"
         );
 
         assertEquals(
-            "SELECT id FROM users /* outer /* $8 inner */ WHERE id = ?",
+            "SELECT id FROM users /* outer /* $8 inner */ $9 still comment */ WHERE id = ?",
             parsedSql.parameters().executableSql()
         );