diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 13f4205b10..07d5cf0278 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -32,6 +32,22 @@ jobs: run: zig build fuzz-cortheon - name: Served Flash Next gate regressions run: python3 -m unittest discover -s tools/zig -p flashnext_batch_serve_test.py + # The generated Engram tables: committing a table that its generator no longer reproduces is a silent drift. + # The generator refuses an interpreter whose unicodedata is not the pinned 16.0.0, which 3.14 carries and 3.12/3.13 do not. + - uses: actions/setup-python@v5 + id: nfkd-python + with: + python-version: "3.14" + - name: NFKD tables are current + env: + PYTHON: ${{ steps.nfkd-python.outputs.python-path }} + run: | + "$PYTHON" tools/zig/nfkd_tables.py --check --zig "$(command -v zig)" || + { echo "regen: python3 tools/zig/nfkd_tables.py --zig zig"; exit 1; } + - name: NFKD generator tests + env: + PYTHON: ${{ steps.nfkd-python.outputs.python-path }} + run: cd tools/zig && "$PYTHON" -m unittest test_nfkd_tables # The lean rule (600-line files, one-line comments within 120 columns): no problem the base branch lacks. - name: lean check env: diff --git a/tools/zig/nfkd_tables.py b/tools/zig/nfkd_tables.py new file mode 100644 index 0000000000..b9650e2c5e --- /dev/null +++ b/tools/zig/nfkd_tables.py @@ -0,0 +1,97 @@ +"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source.""" +import argparse +import subprocess +import sys +import tempfile +import unicodedata +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig" +VERSION = "16.0.0" +ZIG = "zig" + + +def rows(values, per_line, fmt): + """Comma-terminated lines of at most `per_line` values, four-space indented.""" + for i in range(0, len(values), per_line): + yield " " + " ".join(fmt(v) + "," for v in values[i:i + per_line]) + + +def render(): + """The generated Zig source before the compiler's formatter aligns the columns.""" + flat, decompositions = [], [] + for cp in range(0x110000): + decomposed = unicodedata.normalize("NFKD", chr(cp)) + if decomposed != chr(cp): + decompositions.append((cp, len(flat), len(decomposed))) + flat.extend(ord(c) for c in decomposed) + marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")] + lower_flat, lowercases = [], [] + for cp in range(0x110000): + low = chr(cp).lower() + if low != chr(cp): + lowercases.append((cp, len(lower_flat), len(low))) + lower_flat.extend(ord(c) for c in low) + cccs = [] + for cp in range(0x110000): + ccc = unicodedata.combining(chr(cp)) + if not ccc: + continue + if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp: + cccs[-1][1] = cp + else: + cccs.append([cp, cp, ccc]) + decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e + ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t) + return "\n".join([ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION, + 'pub const version = "%s";' % VERSION, + "", + "pub const Decomp = struct { cp: u21, start: u32, len: u8 };", + "", + "// All NFKD sequences concatenated, indexed by Decomp.start/.len.", + "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};", + "", + "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.", + "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const Lower = struct { cp: u21, start: u32, len: u8 };", + "", + "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).", + "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};", + "", + "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION, + "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};", + ]) + "\n" + + +def formatted(text, zig): + """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped.""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "nfkd_data.zig" + path.write_text(text) + subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True) + return path.read_text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the committed file differs") + parser.add_argument("--zig", default=ZIG, help="the Zig compiler whose formatter shapes the output") + args = parser.parse_args() + if unicodedata.unidata_version != VERSION: + sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION)) + text = formatted(render(), args.zig) + if args.check: + sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT)) + OUT.write_text(text) + print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION)) + + +if __name__ == "__main__": + main() diff --git a/tools/zig/test_nfkd_tables.py b/tools/zig/test_nfkd_tables.py new file mode 100644 index 0000000000..7b2877eec8 --- /dev/null +++ b/tools/zig/test_nfkd_tables.py @@ -0,0 +1,47 @@ +"""The generated Engram NFKD tables reproduce, and refuse to build on the wrong Unicode.""" +import sys +import unittest +from unittest.mock import patch + +import nfkd_tables as tables + + +class GeneratorTests(unittest.TestCase): + def test_the_committed_table_is_what_the_generator_renders(self): + rendered = tables.formatted(tables.render(), tables.ZIG) + self.assertEqual(tables.OUT.read_text(), rendered, + "nfkd_data.zig is stale; rerun tools/zig/nfkd_tables.py --zig zig") + + def test_the_header_names_the_generator_that_is_committed(self): + head = tables.OUT.read_text().splitlines()[:2] + self.assertEqual(head, [ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." + % tables.VERSION, + 'pub const version = "%s";' % tables.VERSION, + ]) + self.assertTrue((tables.ROOT / "tools" / "zig" / "nfkd_tables.py").is_file()) + + def test_the_tables_match_the_pinned_unicode_version(self): + self.assertEqual(tables.VERSION, "16.0.0") + if tables.unicodedata.unidata_version != tables.VERSION: + self.skipTest("this interpreter has Unicode %s" % tables.unicodedata.unidata_version) + self.assertEqual(tables.unicodedata.unidata_version, tables.VERSION) + + def test_a_wrong_unicode_version_is_refused_rather_than_emitted(self): + with patch.object(tables.unicodedata, "unidata_version", "15.0.0"): + with patch.object(sys, "argv", ["nfkd_tables.py", "--check"]): + with self.assertRaises(SystemExit) as raised: + tables.main() + self.assertIn("need 16.0.0", str(raised.exception.code)) + + def test_the_guard_runs_before_the_table_is_written(self): + before = tables.OUT.read_text() + with patch.object(tables.unicodedata, "unidata_version", "15.0.0"): + with patch.object(sys, "argv", ["nfkd_tables.py"]): + with self.assertRaises(SystemExit): + tables.main() + self.assertEqual(tables.OUT.read_text(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/zig/src/core/tokenizer/nfkd_data.zig b/zig/src/core/tokenizer/nfkd_data.zig index 1bc3595eb2..38864b3c76 100644 --- a/zig/src/core/tokenizer/nfkd_data.zig +++ b/zig/src/core/tokenizer/nfkd_data.zig @@ -1,10 +1,9 @@ -//! NFKD data for the Engram token-map normalizer chain, generated from -//! unicodedata 16.0.0 (the version unicode_data.zig pins). Fork tool: -//! .agents/tools/gen_nfkd_tables.py regenerates this file. +// Generated by tools/zig/nfkd_tables.py from the Unicode 16.0.0 character database; do not edit. +pub const version = "16.0.0"; pub const Decomp = struct { cp: u21, start: u32, len: u8 }; -/// All NFKD sequences concatenated, indexed by Decomp.start/.len. +// All NFKD sequences concatenated, indexed by Decomp.start/.len. pub const sequences = [_]u21{ 0x0020, 0x0020, 0x0308, 0x0061, 0x0020, 0x0304, 0x0032, 0x0033, 0x0020, 0x0301, 0x03BC, 0x0020, 0x0327, 0x0031, 0x006F, 0x0031, @@ -22385,8 +22384,7 @@ pub const decompositions = [_]Decomp{ .{ .cp = 0x2FA1D, .start = 42308, .len = 1 }, }; -/// General-category Mn/Mc/Me codepoints, sorted; StripAccents removes these -/// (the Rust tokenizers' StripAccents filters all of category M*). +// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*. pub const marks = [_]u21{ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, @@ -22705,7 +22703,7 @@ pub const marks = [_]u21{ pub const Lower = struct { cp: u21, start: u32, len: u8 }; -/// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). +// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). pub const lower_sequences = [_]u21{ 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, @@ -24355,7 +24353,7 @@ pub const lowercases = [_]Lower{ .{ .cp = 0x1E921, .start = 1460, .len = 1 }, }; -/// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. +// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. pub const cccs = [_][3]u21{ .{ 0x0300, 0x0314, 0x00E6 }, .{ 0x0315, 0x0315, 0x00E8 }, .{ 0x0316, 0x0319, 0x00DC }, .{ 0x031A, 0x031A, 0x00E8 }, .{ 0x031B, 0x031B, 0x00D8 }, .{ 0x031C, 0x0320, 0x00DC }, .{ 0x0321, 0x0322, 0x00CA }, .{ 0x0323, 0x0326, 0x00DC },