fix(nl): preserve literal whitespace in NL intent translation (ticket-023)

project/TICKETS.md

@@ -25,4 +25,8 @@ This file indexes governance tickets without taking ownership of
 | **ticket-017** | [`README.md`](./ticket-017/README.md) | - | - |  [`ai-codex.md`](./ticket-017/ai-codex.md) | - | - |
 | **ticket-018** | [`README.md`](./ticket-018/README.md) | - | - | - | - | - |
 | **ticket-019** | [`README.md`](./ticket-019/README.md) | - | - | - | - | - |
+| **ticket-020** | [`README.md`](./ticket-020/README.md) | - | - | - | - | - |
+| **ticket-021** | [`README.md`](./ticket-021/README.md) | - | - | - | - | - |
+| **ticket-022** | [`README.md`](./ticket-022/README.md) | - | - | - | - | - |
+| **ticket-023** | [`README.md`](./ticket-023/README.md) | - | - |  [`ai-codex.md`](./ticket-023/ai-codex.md) | - | - |
 <!-- AUTO:TICKET_INDEX:END -->

project/ticket-023/README.md

@@ -0,0 +1,21 @@
+# ticket-023: Preserve NL literal whitespace
+
+- **ID**: ticket-023
+- **Owner**: agent:codex
+- **Status**: IN_PROGRESS
+- **Workflow state**: EDIT
+- **Created**: 2026-10-02
+
+## Goal and scope
+
+Preserve exact literal arguments while matching normalized NL verb phrases. Keep repeated spaces, tabs and Unicode whitespace in the original remainder passed to existing extractors.
+
+## Acceptance criteria
+
+- [x] AC-01: New regressions fail on the base and pass after correction.
+- [x] AC-02: NL subsystem tests and exact-base governance pass.
+- [x] AC-03: Independent protected review approves and merges the exact head.
+
+## Validation
+
+14 new regressions fail before the correction; 135 NL subsystem tests pass after it. Exact literal spaces, tabs, nonbreaking spaces and Unicode survive recognition into GUI input, CSS selectors and SQL IR. Wellman reports zero errors and warnings. Raw logs remain private; independent publication is pending.

project/ticket-023/ai-codex.md

@@ -0,0 +1 @@
+SESSION_EXECUTION_AUTHORIZATION: user requested merge, testing, repairs and continuation. Scope is the allocated NL parser and its two test modules. Independent approval remains required.

project/ticket-023/intent.json

@@ -0,0 +1,84 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-023",
+  "summary": "Preserve literal whitespace in NL intent translation",
+  "workstream": "core",
+  "classification": {
+    "kind": "BUG",
+    "priority": "P1",
+    "origin": "regression"
+  },
+  "allowedPaths": [
+    "project/ticket-023/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "testql/adapters/nl/intent_recognizer.py",
+    "tests/test_nl_adapter.py",
+    "tests/test_nl_intent_recognizer.py"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null,
+  "delivery": {
+    "acceptedBaseSha": "85caa15e56203d141e0edf69cb426a1430309080",
+    "targetBranch": "main",
+    "outcome": "NL parsing preserves exact whitespace and Unicode inside literal arguments; independently publish the correction.",
+    "nonGoals": [
+      "No new lexicon, models or public API",
+      "Preserve concurrent source work and other tickets"
+    ],
+    "complexity": "S",
+    "estimatedMinutes": 20,
+    "budgets": {
+      "maxImplementationFiles": 3,
+      "maxAffectedComponents": 1,
+      "maxPublicInterfaceChanges": 0,
+      "maxRuntimeDependencies": 0
+    },
+    "architecture": {
+      "status": "accepted",
+      "decision": "Limit raw-line token splitting to the matched verb prefix and preserve the remainder for entity extraction.",
+      "components": [
+        {
+          "name": "nl-adapter",
+          "paths": [
+            "testql/adapters/nl/intent_recognizer.py",
+            "tests/test_nl_adapter.py",
+            "tests/test_nl_intent_recognizer.py"
+          ]
+        }
+      ],
+      "responsibilityChanges": false,
+      "interfaceChanges": [],
+      "dataChanges": [],
+      "ui": {
+        "impact": "none",
+        "states": [],
+        "evidence": []
+      },
+      "rollback": "Revert the bounded parser correction."
+    },
+    "runtimeDependencies": [],
+    "validation": [
+      {
+        "criterion": "AC-01",
+        "commands": [
+          "pytest affected NL tests"
+        ],
+        "evidence": "Regression fails on unchanged parser and passes after correction."
+      },
+      {
+        "criterion": "AC-02",
+        "commands": [
+          "pytest NL subsystem",
+          "project/governance-check.sh"
+        ],
+        "evidence": "Existing NL tests and exact-base scope checks pass."
+      }
+    ]
+  }
+}

testql/adapters/nl/intent_recognizer.py

@@ -18,7 +18,7 @@ class IntentMatch:
 
     intent: str            # navigate | click | input | assert | wait | api | sql | encoder | unknown
     verb: str = ""         # the matched verb phrase (lower-cased)
-    tail: str = ""         # text after the verb (preserves original casing)
+    tail: str = ""         # text after the verb (preserves casing and literal whitespace)
     raw: str = ""          # original line
     confidence: float = 0.0
     extras: dict = field(default_factory=dict)  # e.g. {"prepositions": [...], "field_nouns": [...]}
@@ -53,11 +53,10 @@ def recognize_intent(line: str, lexicon: dict) -> IntentMatch:
                                confidence=1.0, extras=extras)
         prefix = f"{verb} "
         if normalized.startswith(prefix):
-            # Recover original-case tail by chopping the same number of *raw*
-            # tokens off the original line.
+            # Split only the matched verb prefix. Splitting and rejoining the
+            # entire line would also normalize whitespace inside literals.
             verb_word_count = len(verb.split())
-            raw_tokens = line.strip().split()
-            tail = " ".join(raw_tokens[verb_word_count:])
+            tail = line.strip().split(maxsplit=verb_word_count)[verb_word_count]
             return IntentMatch(intent=intent, verb=verb, tail=tail, raw=line,
                                confidence=0.95, extras=extras)
     return IntentMatch(intent="unknown", raw=line, confidence=0.0)

tests/test_nl_adapter.py

@@ -50,6 +50,31 @@
 """
 
 
+@pytest.mark.parametrize("value", ["A  B", "東京\tŁódź", "A\u00a0B"])
+@pytest.mark.parametrize(
+    ("lang", "command"), [("en", "Type"), ("pl", "Wprowadź")]
+)
+def test_input_ir_preserves_literal_whitespace(value, lang, command):
+    plan = parse(f'# SCENARIO: literal\nTYPE: gui\nLANG: {lang}\n1. {command} "{value}"')
+    assert isinstance(plan.steps[0], GuiStep)
+    assert plan.steps[0].action == "input"
+    assert plan.steps[0].value == value
+
+
+def test_click_ir_preserves_whitespace_inside_selector():
+    selector = "[data-name='A  B']"
+    plan = parse(f"# SCENARIO: selector\nTYPE: gui\nLANG: en\n1. Click `{selector}`")
+    assert isinstance(plan.steps[0], GuiStep)
+    assert plan.steps[0].selector == selector
+
+
+def test_sql_ir_preserves_query_whitespace():
+    query = "SELECT *  FROM users"
+    plan = parse(f"# SCENARIO: query\nTYPE: sql\nLANG: pl\n1. Zapytaj {query}")
+    assert isinstance(plan.steps[0], SqlStep)
+    assert plan.steps[0].query == query
+
+
 class TestDetect:
     def test_detect_by_extension(self, tmp_path: Path):
         p = tmp_path / "x.nl.md"

tests/test_nl_intent_recognizer.py

@@ -100,6 +100,24 @@ def test_wykonaj_zapytanie_sql_beats_wykonaj(self, pl):
         assert m.intent == "sql"
 
 
+@pytest.mark.parametrize(
+    ("lang", "line", "tail"),
+    [
+        ("en", 'Type "A  B" into field', '"A  B" into field'),
+        ("pl", 'Wprowadź "東京\tŁódź" do pola email', '"東京\tŁódź" do pola email'),
+        ("pl", 'Wprowadź "A\u00a0B" do pola email', '"A\u00a0B" do pola email'),
+        ("en", "Click `[data-name='A  B']`", "`[data-name='A  B']`"),
+        ("en", "  Go\tto   `/path with  spaces`  ", "`/path with  spaces`"),
+        ("pl", "Wykonaj\tzapytanie  SQL SELECT *  FROM users", "SELECT *  FROM users"),
+    ],
+)
+def test_recognition_preserves_original_argument_whitespace(lang, line, tail):
+    match = recognize_intent(line, load_lexicon(lang))
+    assert match.intent != "unknown"
+    assert match.tail == tail
+    assert match.raw == line
+
+
 class TestRecognizeOperator:
     def test_equal_pl(self, pl):
         op = recognize_operator("status to 200", pl)