fix: parse heredocs with a single-character delimiter (#314)

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Fixed
 
 - Restore `py.typed` marker so type checkers recognize `hcl2` (and `cli`) as typed packages. ([#298](https://github.com/amplify-education/python-hcl2/issues/298))
+- Parse heredocs whose delimiter is a single character, such as `<<E`. The spec defines the delimiter as an Identifier, which permits one character. ([#314](https://github.com/amplify-education/python-hcl2/issues/314))
 - Parse heredocs with an empty body again. A marker immediately followed by its closing delimiter failed to match, and the lexer then ran on to a later delimiter, silently absorbing the attributes in between. ([#309](https://github.com/amplify-education/python-hcl2/issues/309))
 - Negative integer literals load as numbers again instead of `${-N}` expression strings, matching negative floats and the pre-8.x behaviour. ([#307](https://github.com/amplify-education/python-hcl2/issues/307))
 - `strip_string_quotes` no longer unquotes string literals nested inside expressions, which produced invalid HCL such as `${upper(x)}` from `upper("x")`. ([#310](https://github.com/amplify-education/python-hcl2/issues/310))

hcl2/hcl2.lark

@@ -86,9 +86,11 @@ COLONS: "::"
 // it applies between tokens, never inside a terminal's own pattern, so without
 // this a heredoc in a CRLF file fails to match at all. The body group is lazy
 // and optional so an empty body matches without the delimiter search running on
-// to a later marker.
-HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/
-HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/
+// to a later marker. The delimiter itself is `[a-zA-Z][a-zA-Z0-9._-]*` — the
+// trailing `*` rather than `+` because the spec's Identifier permits a single
+// character, so `<<E` is valid.
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/
 
 // Ignore whitespace (but not newlines, as they're significant in HCL).
 // \r is ignored too so CRLF line endings (\r\n) parse the same as LF: the

hcl2/utils.py

@@ -5,11 +5,13 @@
 from dataclasses import dataclass, replace
 from typing import Optional, Tuple
 
-# \r? mirrors the heredoc terminals in hcl2.lark: these run against a token the
-# grammar already accepted, so failing to match a CRLF heredoc here would raise
-# on input that parsed cleanly.
-HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S)
-HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S)
+# These mirror the heredoc terminals in hcl2.lark and must track them. They run
+# against a token the grammar has already accepted, so any delimiter or line
+# ending the grammar admits but these reject raises RuntimeError on input that
+# parsed cleanly: hence `\r?` for CRLF, and `*` rather than `+` so a
+# single-character delimiter is matched here too.
+HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]*)\r?\n([\s\S]*)\1", re.S)
+HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]*)\r?\n([\s\S]*)\1", re.S)
 
 
 @dataclass

test/unit/test_api.py

@@ -534,3 +534,36 @@ def test_escaped_interpolation_marker_is_left_alone(self):
     def test_default_options_still_preserve_source_form(self):
         """Without the option, the source form is kept for reconstruction."""
         self.assertEqual(loads(r'a = "line1\nline2"' + "\n"), {"a": r'"line1\nline2"'})
+
+
+class TestSingleCharacterHeredocDelimiter(TestCase):
+    """`<<E` is a valid heredoc; the delimiter may be one character.
+
+    The spec defines the delimiter as an Identifier — `ID_Start (ID_Continue |
+    '-')*` — whose trailing `*` permits a single character. The grammar used
+    `+`, which required a second one, so `<<E` fell through to STRING_CHARS and
+    the parse failed.
+    """
+
+    def test_single_character_delimiter(self):
+        self.assertEqual(loads("a = <<E\nx\nE\n"), {"a": '"<<E\nx\nE"'})
+
+    def test_single_character_delimiter_trimmed(self):
+        self.assertEqual(loads("a = <<-E\n  x\n  E\n"), {"a": '"<<-E\n  x\n  E"'})
+
+    def test_single_character_delimiter_empty_body(self):
+        self.assertEqual(loads("a = <<E\nE\n"), {"a": '"<<E\nE"'})
+
+    def test_single_character_delimiter_flattens(self):
+        options = SerializationOptions(preserve_heredocs=False)
+        self.assertEqual(loads("a = <<E\nx\nE\n", serialization_options=options), {"a": '"x"'})
+
+    def test_single_character_delimiter_does_not_swallow_what_follows(self):
+        self.assertEqual(loads("a = <<E\nx\nE\nb = 1\n"), {"a": '"<<E\nx\nE"', "b": 1})
+
+    def test_body_line_ending_in_the_delimiter_still_safe(self):
+        """A one-character delimiter makes an accidental match likelier."""
+        self.assertEqual(loads("a = <<E\nsayE\nE\n"), {"a": '"<<E\nsayE\nE"'})
+
+    def test_multi_character_delimiter_unaffected(self):
+        self.assertEqual(loads("a = <<EOF\nx\nEOF\n"), {"a": '"<<EOF\nx\nEOF"'})