tokenizer: commit the Engram NFKD table's generator and check it in CI

.github/workflows/no-gpu.yml

@@ -32,6 +32,22 @@ jobs:
         run: zig build fuzz-cortheon
       - name: Served Flash Next gate regressions
         run: python3 -m unittest discover -s tools/zig -p flashnext_batch_serve_test.py
+      # The generated Engram tables: committing a table that its generator no longer reproduces is a silent drift.
+      # The generator refuses an interpreter whose unicodedata is not the pinned 16.0.0, which 3.14 carries and 3.12/3.13 do not.
+      - uses: actions/setup-python@v5
+        id: nfkd-python
+        with:
+          python-version: "3.14"
+      - name: NFKD tables are current
+        env:
+          PYTHON: ${{ steps.nfkd-python.outputs.python-path }}
+        run: |
+          "$PYTHON" tools/zig/nfkd_tables.py --check --zig "$(command -v zig)" ||
+            { echo "regen: python3 tools/zig/nfkd_tables.py --zig zig"; exit 1; }
+      - name: NFKD generator tests
+        env:
+          PYTHON: ${{ steps.nfkd-python.outputs.python-path }}
+        run: cd tools/zig && "$PYTHON" -m unittest test_nfkd_tables
       # The lean rule (600-line files, one-line comments within 120 columns): no problem the base branch lacks.
       - name: lean check
         env:

tools/zig/nfkd_tables.py

@@ -0,0 +1,97 @@
+"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source."""
+import argparse
+import subprocess
+import sys
+import tempfile
+import unicodedata
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig"
+VERSION = "16.0.0"
+ZIG = "zig"
+
+
+def rows(values, per_line, fmt):
+    """Comma-terminated lines of at most `per_line` values, four-space indented."""
+    for i in range(0, len(values), per_line):
+        yield "    " + " ".join(fmt(v) + "," for v in values[i:i + per_line])
+
+
+def render():
+    """The generated Zig source before the compiler's formatter aligns the columns."""
+    flat, decompositions = [], []
+    for cp in range(0x110000):
+        decomposed = unicodedata.normalize("NFKD", chr(cp))
+        if decomposed != chr(cp):
+            decompositions.append((cp, len(flat), len(decomposed)))
+            flat.extend(ord(c) for c in decomposed)
+    marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")]
+    lower_flat, lowercases = [], []
+    for cp in range(0x110000):
+        low = chr(cp).lower()
+        if low != chr(cp):
+            lowercases.append((cp, len(lower_flat), len(low)))
+            lower_flat.extend(ord(c) for c in low)
+    cccs = []
+    for cp in range(0x110000):
+        ccc = unicodedata.combining(chr(cp))
+        if not ccc:
+            continue
+        if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp:
+            cccs[-1][1] = cp
+        else:
+            cccs.append([cp, cp, ccc])
+    decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e
+    ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t)
+    return "\n".join([
+        "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION,
+        'pub const version = "%s";' % VERSION,
+        "",
+        "pub const Decomp = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// All NFKD sequences concatenated, indexed by Decomp.start/.len.",
+        "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};",
+        "",
+        "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.",
+        "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const Lower = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).",
+        "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};",
+        "",
+        "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION,
+        "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};",
+    ]) + "\n"
+
+
+def formatted(text, zig):
+    """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped."""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = Path(tmp) / "nfkd_data.zig"
+        path.write_text(text)
+        subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True)
+        return path.read_text()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="fail if the committed file differs")
+    parser.add_argument("--zig", default=ZIG, help="the Zig compiler whose formatter shapes the output")
+    args = parser.parse_args()
+    if unicodedata.unidata_version != VERSION:
+        sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION))
+    text = formatted(render(), args.zig)
+    if args.check:
+        sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT))
+    OUT.write_text(text)
+    print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION))
+
+
+if __name__ == "__main__":
+    main()

tools/zig/test_nfkd_tables.py

@@ -0,0 +1,47 @@
+"""The generated Engram NFKD tables reproduce, and refuse to build on the wrong Unicode."""
+import sys
+import unittest
+from unittest.mock import patch
+
+import nfkd_tables as tables
+
+
+class GeneratorTests(unittest.TestCase):
+    def test_the_committed_table_is_what_the_generator_renders(self):
+        rendered = tables.formatted(tables.render(), tables.ZIG)
+        self.assertEqual(tables.OUT.read_text(), rendered,
+                         "nfkd_data.zig is stale; rerun tools/zig/nfkd_tables.py --zig zig")
+
+    def test_the_header_names_the_generator_that_is_committed(self):
+        head = tables.OUT.read_text().splitlines()[:2]
+        self.assertEqual(head, [
+            "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit."
+            % tables.VERSION,
+            'pub const version = "%s";' % tables.VERSION,
+        ])
+        self.assertTrue((tables.ROOT / "tools" / "zig" / "nfkd_tables.py").is_file())
+
+    def test_the_tables_match_the_pinned_unicode_version(self):
+        self.assertEqual(tables.VERSION, "16.0.0")
+        if tables.unicodedata.unidata_version != tables.VERSION:
+            self.skipTest("this interpreter has Unicode %s" % tables.unicodedata.unidata_version)
+        self.assertEqual(tables.unicodedata.unidata_version, tables.VERSION)
+
+    def test_a_wrong_unicode_version_is_refused_rather_than_emitted(self):
+        with patch.object(tables.unicodedata, "unidata_version", "15.0.0"):
+            with patch.object(sys, "argv", ["nfkd_tables.py", "--check"]):
+                with self.assertRaises(SystemExit) as raised:
+                    tables.main()
+        self.assertIn("need 16.0.0", str(raised.exception.code))
+
+    def test_the_guard_runs_before_the_table_is_written(self):
+        before = tables.OUT.read_text()
+        with patch.object(tables.unicodedata, "unidata_version", "15.0.0"):
+            with patch.object(sys, "argv", ["nfkd_tables.py"]):
+                with self.assertRaises(SystemExit):
+                    tables.main()
+        self.assertEqual(tables.OUT.read_text(), before)
+
+
+if __name__ == "__main__":
+    unittest.main()

zig/src/core/tokenizer/nfkd_data.zig

@@ -1,10 +1,9 @@
-//! NFKD data for the Engram token-map normalizer chain, generated from
-//! unicodedata 16.0.0 (the version unicode_data.zig pins). Fork tool:
-//! .agents/tools/gen_nfkd_tables.py regenerates this file.
+// Generated by tools/zig/nfkd_tables.py from the Unicode 16.0.0 character database; do not edit.
+pub const version = "16.0.0";
 
 pub const Decomp = struct { cp: u21, start: u32, len: u8 };
 
-/// All NFKD sequences concatenated, indexed by Decomp.start/.len.
+// All NFKD sequences concatenated, indexed by Decomp.start/.len.
 pub const sequences = [_]u21{
     0x0020,  0x0020,  0x0308,  0x0061,  0x0020,  0x0304,  0x0032,  0x0033,
     0x0020,  0x0301,  0x03BC,  0x0020,  0x0327,  0x0031,  0x006F,  0x0031,
@@ -22385,8 +22384,7 @@ pub const decompositions = [_]Decomp{
     .{ .cp = 0x2FA1D, .start = 42308, .len = 1 },
 };
 
-/// General-category Mn/Mc/Me codepoints, sorted; StripAccents removes these
-/// (the Rust tokenizers' StripAccents filters all of category M*).
+// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.
 pub const marks = [_]u21{
     0x0300,  0x0301,  0x0302,  0x0303,  0x0304,  0x0305,  0x0306,  0x0307,
     0x0308,  0x0309,  0x030A,  0x030B,  0x030C,  0x030D,  0x030E,  0x030F,
@@ -22705,7 +22703,7 @@ pub const marks = [_]u21{
 
 pub const Lower = struct { cp: u21, start: u32, len: u8 };
 
-/// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).
+// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).
 pub const lower_sequences = [_]u21{
     0x0061,  0x0062,  0x0063,  0x0064,  0x0065,  0x0066,  0x0067,  0x0068,
     0x0069,  0x006A,  0x006B,  0x006C,  0x006D,  0x006E,  0x006F,  0x0070,
@@ -24355,7 +24353,7 @@ pub const lowercases = [_]Lower{
     .{ .cp = 0x1E921, .start = 1460, .len = 1 },
 };
 
-/// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0.
+// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0.
 pub const cccs = [_][3]u21{
     .{ 0x0300, 0x0314, 0x00E6 },   .{ 0x0315, 0x0315, 0x00E8 },   .{ 0x0316, 0x0319, 0x00DC },   .{ 0x031A, 0x031A, 0x00E8 },
     .{ 0x031B, 0x031B, 0x00D8 },   .{ 0x031C, 0x0320, 0x00DC },   .{ 0x0321, 0x0322, 0x00CA },   .{ 0x0323, 0x0326, 0x00DC },