From 0b5525acba372115a17cbadb0b8da68fa774d398 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:20:06 -0700 Subject: [PATCH 1/2] Commit the Engram NFKD table's generator beside the table nfkd_data.zig is 24,459 generated lines whose provenance header named .agents/tools/gen_nfkd_tables.py, a path that is not in this repository. The table could therefore not be regenerated or checked from a clean checkout, and nothing in CI noticed if it drifted from the Unicode database it claims. tools/zig/nfkd_tables.py reproduces the committed table exactly: the regenerated file differs only in its header and in reflowed comments, and the sole new declaration is `pub const version`, which unicode_data.zig also carries. It follows unicode_tables.py's conventions, including refusing to run on an interpreter whose unicodedata is not 16.0.0. The check runs in the no-GPU workflow, and tools/zig/test_nfkd_tables.py covers the reproduction, the header, the pinned version and the guard: removing the guard fails two of its five tests. Verified with zig build test (115 binaries, none failing), both generators' --check, and lean_check.py reporting nothing in the touched files. --- .github/workflows/no-gpu.yml | 5 ++ tools/zig/nfkd_tables.py | 97 ++++++++++++++++++++++++++++ tools/zig/test_nfkd_tables.py | 47 ++++++++++++++ zig/src/core/tokenizer/nfkd_data.zig | 14 ++-- 4 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 tools/zig/nfkd_tables.py create mode 100644 tools/zig/test_nfkd_tables.py diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 13f4205b10..11db748500 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -32,6 +32,11 @@ jobs: run: zig build fuzz-cortheon - name: Served Flash Next gate regressions run: python3 -m unittest discover -s tools/zig -p flashnext_batch_serve_test.py + # The generated Engram tables: committing a table that its generator no longer reproduces is a silent drift. + - name: NFKD tables are current + run: | + python3 tools/zig/nfkd_tables.py --check --zig "$(command -v zig)" || + { echo "regen: python3 tools/zig/nfkd_tables.py --zig zig"; exit 1; } # The lean rule (600-line files, one-line comments within 120 columns): no problem the base branch lacks. - name: lean check env: diff --git a/tools/zig/nfkd_tables.py b/tools/zig/nfkd_tables.py new file mode 100644 index 0000000000..b9650e2c5e --- /dev/null +++ b/tools/zig/nfkd_tables.py @@ -0,0 +1,97 @@ +"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source.""" +import argparse +import subprocess +import sys +import tempfile +import unicodedata +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig" +VERSION = "16.0.0" +ZIG = "zig" + + +def rows(values, per_line, fmt): + """Comma-terminated lines of at most `per_line` values, four-space indented.""" + for i in range(0, len(values), per_line): + yield " " + " ".join(fmt(v) + "," for v in values[i:i + per_line]) + + +def render(): + """The generated Zig source before the compiler's formatter aligns the columns.""" + flat, decompositions = [], [] + for cp in range(0x110000): + decomposed = unicodedata.normalize("NFKD", chr(cp)) + if decomposed != chr(cp): + decompositions.append((cp, len(flat), len(decomposed))) + flat.extend(ord(c) for c in decomposed) + marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")] + lower_flat, lowercases = [], [] + for cp in range(0x110000): + low = chr(cp).lower() + if low != chr(cp): + lowercases.append((cp, len(lower_flat), len(low))) + lower_flat.extend(ord(c) for c in low) + cccs = [] + for cp in range(0x110000): + ccc = unicodedata.combining(chr(cp)) + if not ccc: + continue + if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp: + cccs[-1][1] = cp + else: + cccs.append([cp, cp, ccc]) + decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e + ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t) + return "\n".join([ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION, + 'pub const version = "%s";' % VERSION, + "", + "pub const Decomp = struct { cp: u21, start: u32, len: u8 };", + "", + "// All NFKD sequences concatenated, indexed by Decomp.start/.len.", + "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};", + "", + "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.", + "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const Lower = struct { cp: u21, start: u32, len: u8 };", + "", + "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).", + "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};", + "", + "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION, + "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};", + ]) + "\n" + + +def formatted(text, zig): + """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped.""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "nfkd_data.zig" + path.write_text(text) + subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True) + return path.read_text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the committed file differs") + parser.add_argument("--zig", default=ZIG, help="the Zig compiler whose formatter shapes the output") + args = parser.parse_args() + if unicodedata.unidata_version != VERSION: + sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION)) + text = formatted(render(), args.zig) + if args.check: + sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT)) + OUT.write_text(text) + print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION)) + + +if __name__ == "__main__": + main() diff --git a/tools/zig/test_nfkd_tables.py b/tools/zig/test_nfkd_tables.py new file mode 100644 index 0000000000..7b2877eec8 --- /dev/null +++ b/tools/zig/test_nfkd_tables.py @@ -0,0 +1,47 @@ +"""The generated Engram NFKD tables reproduce, and refuse to build on the wrong Unicode.""" +import sys +import unittest +from unittest.mock import patch + +import nfkd_tables as tables + + +class GeneratorTests(unittest.TestCase): + def test_the_committed_table_is_what_the_generator_renders(self): + rendered = tables.formatted(tables.render(), tables.ZIG) + self.assertEqual(tables.OUT.read_text(), rendered, + "nfkd_data.zig is stale; rerun tools/zig/nfkd_tables.py --zig zig") + + def test_the_header_names_the_generator_that_is_committed(self): + head = tables.OUT.read_text().splitlines()[:2] + self.assertEqual(head, [ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." + % tables.VERSION, + 'pub const version = "%s";' % tables.VERSION, + ]) + self.assertTrue((tables.ROOT / "tools" / "zig" / "nfkd_tables.py").is_file()) + + def test_the_tables_match_the_pinned_unicode_version(self): + self.assertEqual(tables.VERSION, "16.0.0") + if tables.unicodedata.unidata_version != tables.VERSION: + self.skipTest("this interpreter has Unicode %s" % tables.unicodedata.unidata_version) + self.assertEqual(tables.unicodedata.unidata_version, tables.VERSION) + + def test_a_wrong_unicode_version_is_refused_rather_than_emitted(self): + with patch.object(tables.unicodedata, "unidata_version", "15.0.0"): + with patch.object(sys, "argv", ["nfkd_tables.py", "--check"]): + with self.assertRaises(SystemExit) as raised: + tables.main() + self.assertIn("need 16.0.0", str(raised.exception.code)) + + def test_the_guard_runs_before_the_table_is_written(self): + before = tables.OUT.read_text() + with patch.object(tables.unicodedata, "unidata_version", "15.0.0"): + with patch.object(sys, "argv", ["nfkd_tables.py"]): + with self.assertRaises(SystemExit): + tables.main() + self.assertEqual(tables.OUT.read_text(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/zig/src/core/tokenizer/nfkd_data.zig b/zig/src/core/tokenizer/nfkd_data.zig index 1bc3595eb2..38864b3c76 100644 --- a/zig/src/core/tokenizer/nfkd_data.zig +++ b/zig/src/core/tokenizer/nfkd_data.zig @@ -1,10 +1,9 @@ -//! NFKD data for the Engram token-map normalizer chain, generated from -//! unicodedata 16.0.0 (the version unicode_data.zig pins). Fork tool: -//! .agents/tools/gen_nfkd_tables.py regenerates this file. +// Generated by tools/zig/nfkd_tables.py from the Unicode 16.0.0 character database; do not edit. +pub const version = "16.0.0"; pub const Decomp = struct { cp: u21, start: u32, len: u8 }; -/// All NFKD sequences concatenated, indexed by Decomp.start/.len. +// All NFKD sequences concatenated, indexed by Decomp.start/.len. pub const sequences = [_]u21{ 0x0020, 0x0020, 0x0308, 0x0061, 0x0020, 0x0304, 0x0032, 0x0033, 0x0020, 0x0301, 0x03BC, 0x0020, 0x0327, 0x0031, 0x006F, 0x0031, @@ -22385,8 +22384,7 @@ pub const decompositions = [_]Decomp{ .{ .cp = 0x2FA1D, .start = 42308, .len = 1 }, }; -/// General-category Mn/Mc/Me codepoints, sorted; StripAccents removes these -/// (the Rust tokenizers' StripAccents filters all of category M*). +// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*. pub const marks = [_]u21{ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, @@ -22705,7 +22703,7 @@ pub const marks = [_]u21{ pub const Lower = struct { cp: u21, start: u32, len: u8 }; -/// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). +// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). pub const lower_sequences = [_]u21{ 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, @@ -24355,7 +24353,7 @@ pub const lowercases = [_]Lower{ .{ .cp = 0x1E921, .start = 1460, .len = 1 }, }; -/// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. +// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. pub const cccs = [_][3]u21{ .{ 0x0300, 0x0314, 0x00E6 }, .{ 0x0315, 0x0315, 0x00E8 }, .{ 0x0316, 0x0319, 0x00DC }, .{ 0x031A, 0x031A, 0x00E8 }, .{ 0x031B, 0x031B, 0x00D8 }, .{ 0x031C, 0x0320, 0x00DC }, .{ 0x0321, 0x0322, 0x00CA }, .{ 0x0323, 0x0326, 0x00DC }, From f2ef564038f5b627571462acc0584454397d1a0b Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:32:32 -0700 Subject: [PATCH 2/2] Run the table check on an interpreter that has the pinned Unicode ubuntu-latest's python3 is 3.12, whose unicodedata is 15.0.0, so the generator's version guard refused to run rather than emit tables from the wrong UCD release. The guard behaved correctly; the workflow assumed an interpreter it did not have. setup-python supplies 3.14, which carries 16.0.0, and the step reaches it by its python-path output so later steps keep their own python3. The generator's tests now run in the same job against the same interpreter. --- .github/workflows/no-gpu.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 11db748500..07d5cf0278 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -33,10 +33,21 @@ jobs: - name: Served Flash Next gate regressions run: python3 -m unittest discover -s tools/zig -p flashnext_batch_serve_test.py # The generated Engram tables: committing a table that its generator no longer reproduces is a silent drift. + # The generator refuses an interpreter whose unicodedata is not the pinned 16.0.0, which 3.14 carries and 3.12/3.13 do not. + - uses: actions/setup-python@v5 + id: nfkd-python + with: + python-version: "3.14" - name: NFKD tables are current + env: + PYTHON: ${{ steps.nfkd-python.outputs.python-path }} run: | - python3 tools/zig/nfkd_tables.py --check --zig "$(command -v zig)" || + "$PYTHON" tools/zig/nfkd_tables.py --check --zig "$(command -v zig)" || { echo "regen: python3 tools/zig/nfkd_tables.py --zig zig"; exit 1; } + - name: NFKD generator tests + env: + PYTHON: ${{ steps.nfkd-python.outputs.python-path }} + run: cd tools/zig && "$PYTHON" -m unittest test_nfkd_tables # The lean rule (600-line files, one-line comments within 120 columns): no problem the base branch lacks. - name: lean check env: