Fix CRLF (\r\n) line endings failing to parse

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Negative integer literals load as numbers again instead of `${-N}` expression strings, matching negative floats and the pre-8.x behaviour. ([#307](https://github.com/amplify-education/python-hcl2/issues/307))
 - `strip_string_quotes` no longer unquotes string literals nested inside expressions, which produced invalid HCL such as `${upper(x)}` from `upper("x")`. ([#310](https://github.com/amplify-education/python-hcl2/issues/310))
 - `strip_string_quotes` now resolves escape sequences, so the values it yields match what the option documents. Escapes naming a codepoint outside the Unicode range, or a lone surrogate, are preserved verbatim rather than raising. ([#308](https://github.com/amplify-education/python-hcl2/issues/308))
+- Parse files with CRLF (`\r\n`) line endings, including heredocs. A `\r` acting as part of a line ending is ignored, so a CRLF file reconstructs with LF endings; a `\r` that is content — inside a quoted string or a heredoc body — is preserved. ([#315](https://github.com/amplify-education/python-hcl2/issues/315))
 
 ## \[8.1.2\] - 2026-04-10
 

hcl2/hcl2.lark

@@ -82,11 +82,23 @@ ELLIPSIS : "..."
 COLONS: "::"
 
 // Heredocs
-HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc)\n/
-HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc_trim)\n/
-
-// Ignore whitespace (but not newlines, as they're significant in HCL)
-%ignore /[ \t]+/
+// \r? accepts CRLF line endings around the markers. %ignore cannot help here:
+// it applies between tokens, never inside a terminal's own pattern, so without
+// this a heredoc in a CRLF file fails to match at all. The body group is lazy
+// and optional so an empty body matches without the delimiter search running on
+// to a later marker.
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/
+
+// Ignore whitespace (but not newlines, as they're significant in HCL).
+// \r is ignored too so CRLF line endings (\r\n) parse the same as LF: the
+// \r before a significant \n is swallowed here, leaving NL_OR_COMMENT to
+// match the \n as usual. A \r that belongs to a terminal's own content is
+// unaffected, because that terminal matches a longer span from the same
+// position and wins: STRING_CHARS keeps a literal CR inside a quoted string,
+// and heredoc bodies keep theirs. Structural \r around heredoc markers is
+// handled by the terminals above, not here.
+%ignore /[ \t\r]+/
 
 // ============================================================================
 // Rules

hcl2/rules/strings.py

@@ -127,7 +127,9 @@ class HeredocTemplateRule(LarkRule):
     """Rule for heredoc template strings (<<MARKER)."""
 
     _children_layout: Tuple[HEREDOC_TEMPLATE]
-    _trim_chars = "\n\t "
+    # \r is trimmed alongside \n so a CRLF heredoc does not leave a stray
+    # carriage return hanging off the closing marker.
+    _trim_chars = "\r\n\t "
 
     @staticmethod
     def lark_name() -> str:

hcl2/utils.py

@@ -5,8 +5,11 @@
 from dataclasses import dataclass, replace
 from typing import Optional, Tuple
 
-HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
-HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
+# \r? mirrors the heredoc terminals in hcl2.lark: these run against a token the
+# grammar already accepted, so failing to match a CRLF heredoc here would raise
+# on input that parsed cleanly.
+HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S)
+HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S)
 
 
 @dataclass

test/unit/test_crlf.py

@@ -0,0 +1,130 @@
+# pylint: disable=C0103,C0114,C0115,C0116
+r"""Regression tests for GH issue #315: CRLF (\r\n) line endings fail to parse.
+
+`hcl2.lark`'s `%ignore` rule only skipped spaces and tabs, so a bare `\r`
+preceding the newline in a CRLF-terminated line had no terminal that could
+consume it: `NL_OR_COMMENT` only matches starting from `\n`. The `\r` fell
+through to `STRING_CHARS` and the parse failed with `UnexpectedToken`, for
+every construct, as soon as a single CRLF line appeared anywhere.
+
+Line endings are handled in three places, so the tests here cross module
+boundaries rather than mirroring one: `%ignore` and the heredoc terminals in
+`hcl2.lark`, the heredoc patterns in `hcl2/utils.py`, and the trim characters
+in `hcl2/rules/strings.py`.
+"""
+
+from unittest import TestCase
+
+from hcl2.api import loads, parses_to_tree, reconstruct, transform
+from hcl2.utils import SerializationOptions
+
+CR = "\r"
+
+
+class TestCrlfLineEndings(TestCase):
+    """A CRLF source parses to the same dict as its LF equivalent."""
+
+    def test_bare_attribute(self):
+        self.assertEqual(loads("a = 1\r\n"), loads("a = 1\n"))
+
+    def test_block(self):
+        crlf = loads("locals {\r\n  a = 1\r\n}\r\n")
+        lf = loads("locals {\n  a = 1\n}\n")
+        self.assertEqual(crlf, lf)
+
+    def test_quoted_string(self):
+        crlf = loads('a = "x"\r\n')
+        lf = loads('a = "x"\n')
+        self.assertEqual(crlf, lf)
+
+    def test_single_crlf_line_amid_lf_lines(self):
+        crlf = loads("a = 1\nb = 2\r\nc = 3\n")
+        lf = loads("a = 1\nb = 2\nc = 3\n")
+        self.assertEqual(crlf, lf)
+
+    def test_tuple_and_object(self):
+        crlf = loads("a = [1,\r\n2]\r\nb = {\r\n  x = 1\r\n}\r\n")
+        lf = loads("a = [1,\n2]\nb = {\n  x = 1\n}\n")
+        self.assertEqual(crlf, lf)
+
+    def test_comments(self):
+        for source in ("# hi\r\na = 1\r\n", "// hi\r\na = 1\r\n", "/* hi */\r\na = 1\r\n"):
+            with self.subTest(source=source):
+                options = SerializationOptions(with_comments=True)
+                result = loads(source, serialization_options=options)
+                self.assertEqual(result["a"], 1)
+                self.assertEqual(result["__comments__"], [{"value": "hi"}])
+
+
+class TestCrlfDoesNotEatContentCarriageReturns(TestCase):
+    r"""Only a `\r` that is insignificant whitespace between tokens is ignored.
+
+    A `\r` inside a terminal's own content belongs to that terminal, which
+    matches a longer span from the same position and so wins.
+    """
+
+    def test_real_cr_inside_a_quoted_string_survives(self):
+        result = loads('a = "x' + CR + 'y"\n')
+        self.assertEqual(result, {"a": '"x' + CR + 'y"'})
+
+    def test_string_consisting_only_of_a_cr_survives(self):
+        self.assertEqual(loads('a = "' + CR + '"\n'), {"a": '"' + CR + '"'})
+
+    def test_cr_escape_sequence_is_untouched(self):
+        r"""`\r` as escape *text* is two ASCII characters, never at risk."""
+        self.assertEqual(loads(r'a = "line1\r\nline2"' + "\n"), {"a": r'"line1\r\nline2"'})
+
+    def test_heredoc_body_keeps_its_carriage_returns(self):
+        r"""HCL treats `\r` around markers as structure but body text as content."""
+        result = loads("a = <<EOF\r\nx\r\ny\r\nEOF\r\n")
+        self.assertEqual(result, {"a": '"<<EOF' + CR + "\nx" + CR + "\ny" + CR + '\nEOF"'})
+
+
+class TestCrlfHeredocs(TestCase):
+    r"""Heredocs need `\r?` in their own terminals; `%ignore` cannot reach inside.
+
+    HCL's reference scanner handles both `\n` and `\r\n` around heredoc markers,
+    so a CRLF heredoc is valid input rather than something to reject.
+    """
+
+    def test_heredoc_parses(self):
+        result = loads("a = <<EOF\r\nx\r\nEOF\r\n")
+        self.assertEqual(result, {"a": '"<<EOF' + CR + "\nx" + CR + '\nEOF"'})
+
+    def test_trimmed_heredoc_parses(self):
+        result = loads("a = <<-EOF\r\n  x\r\n  EOF\r\n")
+        self.assertEqual(result, {"a": '"<<-EOF' + CR + "\n  x" + CR + '\n  EOF"'})
+
+    def test_heredoc_does_not_swallow_the_following_attribute(self):
+        result = loads("a = <<EOF\r\nx\r\nEOF\r\nb = 1\r\n")
+        self.assertEqual(result["b"], 1)
+
+    def test_closing_marker_leaves_no_trailing_carriage_return(self):
+        r"""`_trim_chars` covers `\r`, so the preserved form ends at the marker."""
+        result = loads("a = <<EOF\r\nx\r\nEOF\r\n")
+        self.assertTrue(result["a"].endswith('EOF"'), result["a"])
+
+    def test_flattening_a_crlf_heredoc_does_not_raise(self):
+        """The heredoc patterns in utils.py run on an already-parsed token."""
+        options = SerializationOptions(preserve_heredocs=False)
+        result = loads("a = <<EOF\r\nx\r\ny\r\nEOF\r\n", serialization_options=options)
+        self.assertEqual(result, {"a": '"x' + CR + '\\ny"'})
+
+    def test_trimmed_heredoc_flattens(self):
+        options = SerializationOptions(preserve_heredocs=False)
+        result = loads("a = <<-EOF\r\n  x\r\n  EOF\r\n", serialization_options=options)
+        self.assertEqual(result, {"a": '"x"'})
+
+
+class TestCrlfReconstruction(TestCase):
+    r"""CRLF input reconstructs as LF, because the `\r` never enters the IR."""
+
+    def test_crlf_normalises_to_lf(self):
+        source = "a = 1\r\nb = 2\r\n"
+        output = reconstruct(transform(parses_to_tree(source)).to_lark())
+        self.assertEqual(output, "a = 1\nb = 2\n")
+
+    def test_lf_is_unchanged(self):
+        source = "a = 1\nb = 2\n"
+        output = reconstruct(transform(parses_to_tree(source)).to_lark())
+        self.assertEqual(output, source)