From da765bf1c48db44ad43ed38a0279138324ba047b Mon Sep 17 00:00:00 2001 From: Zakariya Ghazzol Date: Sat, 17 Oct 2026 14:35:57 -0400 Subject: [PATCH 1/3] Preserve quotes and decode string literal escapes --- SPEC.md | 7 ++ src/fpy/syntax.py | 17 ++++- test/fpy/test_string_literals.py | 126 +++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 test/fpy/test_string_literals.py diff --git a/SPEC.md b/SPEC.md index a02698a..f99e88d 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1151,6 +1151,13 @@ Rule: String literals have type [String](#internal-types). +Single-quoted and double-quoted string literals support Python-style backslash +escapes, including escaped quotes and backslashes, control characters, octal, +hexadecimal, and Unicode escapes. Unescaped Unicode characters are preserved. +Malformed hexadecimal, Unicode, and named Unicode escapes are compile errors. +Decoded string values must be representable as UTF-8. Surrogate code points, +including escaped surrogate pairs, are compile errors. + ### Boolean literals #### Syntax `BOOLEAN_LITERAL: "True" | "False"` diff --git a/src/fpy/syntax.py b/src/fpy/syntax.py index 2138602..6f0ecd8 100644 --- a/src/fpy/syntax.py +++ b/src/fpy/syntax.py @@ -1,4 +1,5 @@ from __future__ import annotations +from ast import literal_eval from dataclasses import dataclass, field from enum import Enum from typing import Iterator, List, Literal as TypingLiteral, Union @@ -459,7 +460,21 @@ def wrapper(self, tree): def handle_str(meta, s: str): - return s.strip("'").strip('"') + from fpy.error import SyntaxErrorDuringTransform + + try: + value = literal_eval(s) + value.encode("utf-8") + return value + except SyntaxError as exc: + raise SyntaxErrorDuringTransform( + f"Invalid string literal: {exc.msg}", s + ) from exc + except UnicodeEncodeError as exc: + raise SyntaxErrorDuringTransform( + "Invalid string literal: surrogate code points cannot be encoded as UTF-8", + s, + ) from exc # Check statement clause handlers. diff --git a/test/fpy/test_string_literals.py b/test/fpy/test_string_literals.py new file mode 100644 index 0000000..cc229df --- /dev/null +++ b/test/fpy/test_string_literals.py @@ -0,0 +1,126 @@ +"""String literal values and their exact output on both real sequencers.""" + +import pytest + +from fpy.compiler import text_to_ast +from fpy.dictionary import load_dictionary +from fpy.error import CompileError +from fpy.harness import SequencerHarness +from fpy.syntax import AstString +from fpy.test_helpers import ( + FPYBC, + CompilationFailed, + assert_run_success, + compile_seq, + compile_seq_wasm, + default_dictionary, +) + +STRING_CASES = [ + pytest.param('""', "", id="empty-double"), + pytest.param("''", "", id="empty-single"), + pytest.param("'\"hi\"'", '"hi"', id="preserve-double-quotes"), + pytest.param("\"'hi'\"", "'hi'", id="preserve-single-quotes"), + pytest.param(r'''"say \"hi\""''', 'say "hi"', id="escaped-double-quotes"), + pytest.param(r"'it\'s'", "it's", id="escaped-single-quote"), + pytest.param(r'"a\nb\tc\rd"', "a\nb\tc\rd", id="control-escapes"), + pytest.param(r'"C:\\temp\\"', "C:\\temp\\", id="escaped-backslashes"), + pytest.param(r'"\\n"', r"\n", id="decode-once"), + pytest.param('"héllo✓"', "héllo✓", id="literal-utf8"), + pytest.param(r'"h\u00e9llo\u2713"', "héllo✓", id="unicode-escapes"), + pytest.param(r'"\x41\101"', "AA", id="hex-and-octal"), + pytest.param(r'"\U0001f680"', "🚀", id="long-unicode-escape"), + pytest.param(r'"\N{ROCKET}"', "🚀", id="named-unicode-escape"), +] + +SURROGATE_LITERALS = [ + pytest.param(r'"\ud800"', id="high-surrogate"), + pytest.param(r'"\udfff"', id="low-surrogate"), + pytest.param(r'"\ud83d\ude80"', id="surrogate-pair"), + pytest.param(r'"a\ud800b"', id="embedded-surrogate"), + pytest.param(r"'\ud800'", id="single-quoted-high-surrogate"), + pytest.param(r"'\udfff'", id="single-quoted-low-surrogate"), +] + + +@pytest.mark.parametrize("literal,expected", STRING_CASES) +def test_string_literal_value(literal, expected): + node = text_to_ast(literal).stmts[0] + assert isinstance(node, AstString) + assert node.value == expected + + +@pytest.mark.parametrize( + "literal", + [r'"\x"', r'"\x0g"', r'"\u123"', r'"\U00110000"', r'"\N{NO SUCH NAME}"'], +) +def test_invalid_string_escape_has_source_diagnostic(literal): + with pytest.raises(CompileError, match="Invalid string literal") as exc: + text_to_ast(f"# preceding line\nlog({literal})\n") + assert exc.value.node.line == 2 + assert exc.value.node.column == 5 + assert "Internal error" not in str(exc.value) + + +@pytest.mark.parametrize("literal", SURROGATE_LITERALS) +def test_surrogate_escape_has_source_diagnostic(literal): + with pytest.raises(CompileError, match="Invalid string literal:.*UTF-8") as exc: + text_to_ast(f"# preceding line\nlog({literal})\n") + assert exc.value.node.line == 2 + assert exc.value.node.column == 5 + assert "Internal error" not in str(exc.value) + + +@pytest.mark.parametrize("literal", SURROGATE_LITERALS) +def test_surrogate_escape_is_compile_error(literal, single_backend): + compile_fn = compile_seq if single_backend == FPYBC else compile_seq_wasm + with pytest.raises( + CompilationFailed, match="Invalid string literal:.*UTF-8" + ) as exc: + compile_fn(f"# preceding line\nlog({literal})\n") + assert "Internal error" not in str(exc.value) + + +@pytest.mark.skipif( + "config.getoption('--use-gds')", reason="inspects local sequencer harness output" +) +@pytest.mark.parametrize("literal,expected", STRING_CASES) +def test_string_literal_event_and_command( + literal, expected, single_backend, fprime_test_api, monkeypatch +): + # Observe the actual C++ harness reply; do not replace sequencer execution. + replies = [] + original_run = SequencerHarness.run + + def record_run(harness, request): + result = original_run(harness, request) + replies.append((request, result)) + return result + + monkeypatch.setattr(SequencerHarness, "run", record_run) + assert_run_success( + fprime_test_api, + f"log({literal})\nCdhCore.cmdDisp.CMD_NO_OP_STRING({literal})\n", + ) + assert len(replies) == 1 + request, result = replies[0] + + message = expected + if single_backend == FPYBC: + message = ( + f"(FpySequencer) LogActivityHi : Sequence {request['seqFile']}: {expected}" + ) + assert (5, message) in [ + (event["severity"], event["text"]) for event in result["events"] + ] + + dictionary = load_dictionary(default_dictionary) + types = dictionary["type_defs"] + payload = expected.encode("utf-8") + opcode = dictionary["cmd_name_dict"]["CdhCore.cmdDisp.CMD_NO_OP_STRING"].opcode + expected_command = ( + opcode.to_bytes(types["FwOpcodeType"].max_size, "big") + + len(payload).to_bytes(types["FwSizeStoreType"].max_size, "big") + + payload + ) + assert result["cmds"] == [expected_command.hex()] From e397ceb49ba03173afdade2de967999eb4e69be1 Mon Sep 17 00:00:00 2001 From: Zakariya Ghazzol Date: Mon, 26 Oct 2026 03:21:40 -0400 Subject: [PATCH 2/3] Handle invalid string literal values and leave spec unchanged --- SPEC.md | 7 ------- src/fpy/syntax.py | 2 ++ test/fpy/test_string_literals.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/SPEC.md b/SPEC.md index f99e88d..a02698a 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1151,13 +1151,6 @@ Rule: String literals have type [String](#internal-types). -Single-quoted and double-quoted string literals support Python-style backslash -escapes, including escaped quotes and backslashes, control characters, octal, -hexadecimal, and Unicode escapes. Unescaped Unicode characters are preserved. -Malformed hexadecimal, Unicode, and named Unicode escapes are compile errors. -Decoded string values must be representable as UTF-8. Surrogate code points, -including escaped surrogate pairs, are compile errors. - ### Boolean literals #### Syntax `BOOLEAN_LITERAL: "True" | "False"` diff --git a/src/fpy/syntax.py b/src/fpy/syntax.py index 6f0ecd8..92080f9 100644 --- a/src/fpy/syntax.py +++ b/src/fpy/syntax.py @@ -475,6 +475,8 @@ def handle_str(meta, s: str): "Invalid string literal: surrogate code points cannot be encoded as UTF-8", s, ) from exc + except ValueError as exc: + raise SyntaxErrorDuringTransform(f"Invalid string literal: {exc}", s) from exc # Check statement clause handlers. diff --git a/test/fpy/test_string_literals.py b/test/fpy/test_string_literals.py index cc229df..04fe91b 100644 --- a/test/fpy/test_string_literals.py +++ b/test/fpy/test_string_literals.py @@ -50,6 +50,19 @@ def test_string_literal_value(literal, expected): assert node.value == expected +@pytest.mark.parametrize("quote", ['"', "'"]) +def test_null_byte_has_source_diagnostic(quote): + """Report raw NUL bytes at their source location, not as internal errors.""" + literal = f"{quote}a\0b{quote}" + with pytest.raises( + CompileError, match="Invalid string literal:.*null bytes" + ) as exc: + text_to_ast(f"# preceding line\nlog({literal})\n") + assert exc.value.node.line == 2 + assert exc.value.node.column == 5 + assert "Internal error" not in str(exc.value) + + @pytest.mark.parametrize( "literal", [r'"\x"', r'"\x0g"', r'"\u123"', r'"\U00110000"', r'"\N{NO SUCH NAME}"'], From 68b175061b57ba41f2631ce6cab9aa691bad1b41 Mon Sep 17 00:00:00 2001 From: Zakariya Ghazzol Date: Tue, 27 Oct 2026 03:29:02 -0400 Subject: [PATCH 3/3] Avoid spelling false positives in string diagnostic tests --- test/fpy/test_string_literals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/fpy/test_string_literals.py b/test/fpy/test_string_literals.py index 04fe91b..7566706 100644 --- a/test/fpy/test_string_literals.py +++ b/test/fpy/test_string_literals.py @@ -57,7 +57,7 @@ def test_null_byte_has_source_diagnostic(quote): with pytest.raises( CompileError, match="Invalid string literal:.*null bytes" ) as exc: - text_to_ast(f"# preceding line\nlog({literal})\n") + text_to_ast("# preceding line\n" f"log({literal})\n") assert exc.value.node.line == 2 assert exc.value.node.column == 5 assert "Internal error" not in str(exc.value) @@ -69,7 +69,7 @@ def test_null_byte_has_source_diagnostic(quote): ) def test_invalid_string_escape_has_source_diagnostic(literal): with pytest.raises(CompileError, match="Invalid string literal") as exc: - text_to_ast(f"# preceding line\nlog({literal})\n") + text_to_ast("# preceding line\n" f"log({literal})\n") assert exc.value.node.line == 2 assert exc.value.node.column == 5 assert "Internal error" not in str(exc.value) @@ -78,7 +78,7 @@ def test_invalid_string_escape_has_source_diagnostic(literal): @pytest.mark.parametrize("literal", SURROGATE_LITERALS) def test_surrogate_escape_has_source_diagnostic(literal): with pytest.raises(CompileError, match="Invalid string literal:.*UTF-8") as exc: - text_to_ast(f"# preceding line\nlog({literal})\n") + text_to_ast("# preceding line\n" f"log({literal})\n") assert exc.value.node.line == 2 assert exc.value.node.column == 5 assert "Internal error" not in str(exc.value) @@ -90,7 +90,7 @@ def test_surrogate_escape_is_compile_error(literal, single_backend): with pytest.raises( CompilationFailed, match="Invalid string literal:.*UTF-8" ) as exc: - compile_fn(f"# preceding line\nlog({literal})\n") + compile_fn("# preceding line\n" f"log({literal})\n") assert "Internal error" not in str(exc.value)