From 7fa61c380f05c06391097a54d592fa87531208c5 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Fri, 21 Aug 2026 14:53:23 +0200 Subject: [PATCH 1/6] extra: add dedupe_includes.py Finds files identical across a set of directories and mirrors one canonical copy out, deleting the duplicates from the source dirs. Useful for the generated llext-edk headers duplicated across board variants. Co-Authored-By: Claude Sonnet 5 --- extra/dedupe_includes.py | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 extra/dedupe_includes.py diff --git a/extra/dedupe_includes.py b/extra/dedupe_includes.py new file mode 100755 index 000000000..646f44f01 --- /dev/null +++ b/extra/dedupe_includes.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +# Copyright (c) Arduino s.r.l. and/or its affiliated companies +# SPDX-License-Identifier: Apache-2.0 + +# Finds files that are byte-identical, at the same relative path, across every +# one of a set of input directories, copies one canonical copy of each into a +# separate output directory (preserving its relative path), then deletes the +# duplicate copies from the input directories. +# +# This will require a separate build configuration -- +# re-pointing include search paths at the output directory is a separate, +# manual step. + +import argparse +import hashlib +import sys +from pathlib import Path + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def scan_directory(directory: Path) -> dict[Path, str]: + return { + path.relative_to(directory): hash_file(path) + for path in directory.rglob("*") + if path.is_file() + } + + +def find_common_files(per_dir_hashes: list[dict[Path, str]]) -> dict[Path, str]: + common = {} + first, rest = per_dir_hashes[0], per_dir_hashes[1:] + for rel_path, digest in first.items(): + if all(other.get(rel_path) == digest for other in rest): + common[rel_path] = digest + return common + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--common-dir", + required=True, + type=Path, + help="output directory to mirror common files into; must not already exist", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="report what would be done, without touching the filesystem", + ) + parser.add_argument( + "directories", + nargs="+", + type=Path, + help="two or more directories to compare", + ) + args = parser.parse_args() + + if len(args.directories) < 2: + parser.error("need at least 2 directories to compare") + + if args.common_dir.exists(): + parser.error(f"--common-dir {args.common_dir} already exists") + + per_dir_hashes = [scan_directory(d) for d in args.directories] + common = find_common_files(per_dir_hashes) + + total_size = sum( + (args.directories[0] / rel_path).stat().st_size for rel_path in common + ) + freed = total_size * (len(args.directories) - 1) + + print(f"{len(args.directories)} directories compared") + for d, hashes in zip(args.directories, per_dir_hashes): + print(f" {d}: {len(hashes)} files") + print(f"{len(common)} files common to all directories") + print(f"{freed} bytes would be freed" if args.dry_run else f"{freed} bytes freed") + + if args.dry_run: + return 0 + + for rel_path in common: + src = args.directories[0] / rel_path + dst = args.common_dir / rel_path + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(src.read_bytes()) + if hash_file(dst) != common[rel_path]: + print(f"error: mirrored copy of {rel_path} does not match source", file=sys.stderr) + return 1 + + for rel_path in common: + for d in args.directories: + (d / rel_path).unlink() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 63d6b35fe694fc5998569d6c3f5186c3ef7e8de7 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Fri, 21 Aug 2026 15:14:23 +0200 Subject: [PATCH 2/6] extra/dedupe_includes.py: update variant includes.txt after dedup When applying (not --dry-run), repoint each affected -iwithprefixbefore entry in a variant's includes.txt at the mirrored common directory: fully-emptied entries get replaced (and their now-empty directory removed from disk), partially-emptied entries get an extra entry pointing at the common directory alongside the original. Co-Authored-By: Claude Sonnet 5 --- extra/dedupe_includes.py | 91 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/extra/dedupe_includes.py b/extra/dedupe_includes.py index 646f44f01..70f38f360 100755 --- a/extra/dedupe_includes.py +++ b/extra/dedupe_includes.py @@ -8,15 +8,23 @@ # separate output directory (preserving its relative path), then deletes the # duplicate copies from the input directories. # -# This will require a separate build configuration -- -# re-pointing include search paths at the output directory is a separate, -# manual step. +# If a directory sits under a "variants//" tree with a sibling +# "includes.txt" (a list of "-iwithprefixbefore" compiler flags, each +# path relative to the variant's own directory), that file is updated to +# match: entries fully emptied by the move are repointed at the mirrored +# common directory and their now-empty source directory is deleted; entries +# only partially emptied get an extra entry added for the common directory +# alongside the original. This only happens when not running with --dry-run. import argparse import hashlib +import os +import shutil import sys from pathlib import Path +INCLUDE_FLAG = "-iwithprefixbefore" + def hash_file(path: Path) -> str: digest = hashlib.sha256() @@ -43,6 +51,78 @@ def find_common_files(per_dir_hashes: list[dict[Path, str]]) -> dict[Path, str]: return common +def find_variant_root(path: Path) -> Path | None: + parts = path.parts + for i, part in enumerate(parts): + if part == "variants" and i + 1 < len(parts): + return Path(*parts[: i + 2]) + return None + + +def update_includes_txt( + variant_root: Path, input_dir: Path, common_dir: Path, common_rel_paths +) -> None: + includes_txt = variant_root / "includes.txt" + if not includes_txt.is_file(): + return + + lines = includes_txt.read_text().splitlines() + entries = [ + (i, line[len(INCLUDE_FLAG) :]) + for i, line in enumerate(lines) + if line.startswith(INCLUDE_FLAG) + ] + # Deepest first, so a child directory is resolved (and possibly removed) + # before its parent's emptiness is checked. + entries.sort(key=lambda e: e[1].count("/"), reverse=True) + + actions: dict[int, tuple[str, str]] = {} + for idx, suffix in entries: + real_dir = variant_root / suffix.lstrip("/") + if real_dir != input_dir and input_dir not in (real_dir, *real_dir.parents): + continue + + affected = any( + (input_dir / rel) == real_dir or real_dir in (input_dir / rel).parents + for rel in common_rel_paths + ) + if not affected: + continue + + if real_dir.exists(): + has_files = any(p.is_file() for p in real_dir.rglob("*")) + else: + has_files = False + + sub_path = real_dir.relative_to(input_dir) + common_target = common_dir if sub_path == Path(".") else common_dir / sub_path + new_suffix = "/" + os.path.relpath(common_target, variant_root).replace(os.sep, "/") + new_line = INCLUDE_FLAG + new_suffix + + if not has_files: + if real_dir.exists(): + shutil.rmtree(real_dir) + actions[idx] = ("replace", new_line) + else: + actions[idx] = ("augment", new_line) + + if not actions: + return + + new_lines = [] + for i, line in enumerate(lines): + action = actions.get(i) + if action is None: + new_lines.append(line) + elif action[0] == "replace": + new_lines.append(action[1]) + else: + new_lines.append(line) + new_lines.append(action[1]) + + includes_txt.write_text("\n".join(new_lines) + "\n") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -100,6 +180,11 @@ def main() -> int: for d in args.directories: (d / rel_path).unlink() + for d in args.directories: + variant_root = find_variant_root(d) + if variant_root is not None: + update_includes_txt(variant_root, d, args.common_dir, common.keys()) + return 0 From f1d3eb98bd1268d8c27f1bbc5c6763cc5b959d6b Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Fri, 21 Aug 2026 16:44:38 +0200 Subject: [PATCH 3/6] extra/dedupe_includes.py: take variant names instead of directories Arguments are now variant names (a leading 'variants/' is ignored), resolved to variants//llext-edk/include. --common-dir is also a variant name, created at variants/. Each variant's includes.txt is always updated, replacing the old "variants//" path detection. Co-Authored-By: Claude Opus 5.5 (1M context) --- extra/dedupe_includes.py | 101 +++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/extra/dedupe_includes.py b/extra/dedupe_includes.py index 70f38f360..4b58f0db7 100755 --- a/extra/dedupe_includes.py +++ b/extra/dedupe_includes.py @@ -3,18 +3,19 @@ # Copyright (c) Arduino s.r.l. and/or its affiliated companies # SPDX-License-Identifier: Apache-2.0 -# Finds files that are byte-identical, at the same relative path, across every -# one of a set of input directories, copies one canonical copy of each into a -# separate output directory (preserving its relative path), then deletes the -# duplicate copies from the input directories. +# Finds files that are byte-identical, at the same relative path, in the +# 'llext-edk/include' directory of every given variant, copies one canonical +# copy of each into a common variant's directory (dropping the +# 'llext-edk/include' part of the path), then deletes the duplicate copies +# from the variants' directories. # -# If a directory sits under a "variants//" tree with a sibling -# "includes.txt" (a list of "-iwithprefixbefore" compiler flags, each -# path relative to the variant's own directory), that file is updated to -# match: entries fully emptied by the move are repointed at the mirrored -# common directory and their now-empty source directory is deleted; entries -# only partially emptied get an extra entry added for the common directory -# alongside the original. This only happens when not running with --dry-run. +# Each variant's "includes.txt" (a list of "-iwithprefixbefore" +# compiler flags, each path relative to the variant's own directory) is +# updated to match: entries fully emptied by the move are repointed at the +# common variant's directory and their now-empty source directory is removed +# from disk; entries only partially emptied get an extra entry added for the +# common directory alongside the original. This only happens when not +# running with --dry-run. import argparse import hashlib @@ -24,6 +25,19 @@ from pathlib import Path INCLUDE_FLAG = "-iwithprefixbefore" +VARIANTS_DIR = Path("variants") +LLEXT_INCLUDE = Path("llext-edk/include") + + +def input_path(name: str) -> Path: + return VARIANTS_DIR / name / LLEXT_INCLUDE + + +def strip_variants_prefix(name: str) -> str: + name = name.rstrip("/") + if name.startswith(f"{VARIANTS_DIR}/"): + name = name[len(f"{VARIANTS_DIR}/") :] + return name def hash_file(path: Path) -> str: @@ -51,17 +65,9 @@ def find_common_files(per_dir_hashes: list[dict[Path, str]]) -> dict[Path, str]: return common -def find_variant_root(path: Path) -> Path | None: - parts = path.parts - for i, part in enumerate(parts): - if part == "variants" and i + 1 < len(parts): - return Path(*parts[: i + 2]) - return None - - -def update_includes_txt( - variant_root: Path, input_dir: Path, common_dir: Path, common_rel_paths -) -> None: +def update_includes_txt(name: str, common_dir: Path, common_rel_paths) -> None: + variant_root = VARIANTS_DIR / name + input_dir = input_path(name) includes_txt = variant_root / "includes.txt" if not includes_txt.is_file(): return @@ -128,8 +134,8 @@ def main() -> int: parser.add_argument( "--common-dir", required=True, - type=Path, - help="output directory to mirror common files into; must not already exist", + help="name of the common variant to mirror files into " + "(created at variants/; must not already exist)", ) parser.add_argument( "--dry-run", @@ -137,39 +143,44 @@ def main() -> int: help="report what would be done, without touching the filesystem", ) parser.add_argument( - "directories", + "variant_names", nargs="+", - type=Path, - help="two or more directories to compare", + help="two or more variant names to compare (a leading 'variants/' is ignored)", ) args = parser.parse_args() - if len(args.directories) < 2: - parser.error("need at least 2 directories to compare") + variant_names = [strip_variants_prefix(v) for v in args.variant_names] + if len(variant_names) < 2: + parser.error("need at least 2 variants to compare") + + for name in variant_names: + if not (VARIANTS_DIR / name).is_dir(): + parser.error(f"{VARIANTS_DIR / name} does not exist") - if args.common_dir.exists(): - parser.error(f"--common-dir {args.common_dir} already exists") + common_dir = VARIANTS_DIR / strip_variants_prefix(args.common_dir) + if common_dir.exists(): + parser.error(f"--common-dir {common_dir} already exists") - per_dir_hashes = [scan_directory(d) for d in args.directories] + per_dir_hashes = [scan_directory(input_path(name)) for name in variant_names] common = find_common_files(per_dir_hashes) total_size = sum( - (args.directories[0] / rel_path).stat().st_size for rel_path in common + (input_path(variant_names[0]) / rel_path).stat().st_size for rel_path in common ) - freed = total_size * (len(args.directories) - 1) + freed = total_size * (len(variant_names) - 1) - print(f"{len(args.directories)} directories compared") - for d, hashes in zip(args.directories, per_dir_hashes): - print(f" {d}: {len(hashes)} files") - print(f"{len(common)} files common to all directories") + print(f"{len(variant_names)} variants compared") + for name, hashes in zip(variant_names, per_dir_hashes): + print(f" {input_path(name)}: {len(hashes)} files") + print(f"{len(common)} files common to all variants") print(f"{freed} bytes would be freed" if args.dry_run else f"{freed} bytes freed") if args.dry_run: return 0 for rel_path in common: - src = args.directories[0] / rel_path - dst = args.common_dir / rel_path + src = input_path(variant_names[0]) / rel_path + dst = common_dir / rel_path dst.parent.mkdir(parents=True, exist_ok=True) dst.write_bytes(src.read_bytes()) if hash_file(dst) != common[rel_path]: @@ -177,13 +188,11 @@ def main() -> int: return 1 for rel_path in common: - for d in args.directories: - (d / rel_path).unlink() + for name in variant_names: + (input_path(name) / rel_path).unlink() - for d in args.directories: - variant_root = find_variant_root(d) - if variant_root is not None: - update_includes_txt(variant_root, d, args.common_dir, common.keys()) + for name in variant_names: + update_includes_txt(name, common_dir, common.keys()) return 0 From d1565126186f2599c373f33413ff40794e278b45 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Tue, 13 Oct 2026 16:40:03 +0200 Subject: [PATCH 4/6] package_core: dedupe llext-edk includes in CI When packaging in CI, move the llext-edk headers shared by all the included variants into a 'variants/include' directory via dedupe_includes.py, and add it to the archive. Only variants that actually have an EDK are considered, and the step is skipped when fewer than 2 remain. This modifies the working tree, so it only runs when GITHUB_ACTIONS is set (each packaging job uses a fresh checkout). Since headers may now live outside the variant directory, platform.txt looks up the -imacros files (autoconf.h and zephyr_stdint.h) via the include path instead of a variant-relative path. Co-Authored-By: Claude Opus 5.5 (1M context) --- extra/package_core.sh | 12 ++++++++++++ platform.txt | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extra/package_core.sh b/extra/package_core.sh index 77b431480..5000bf52e 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -88,6 +88,18 @@ for variant in $INCLUDED_VARIANTS ; do fi done +# deduplicate llext-edk headers shared by all included variants +# (CI only: this modifies the working tree) +if [ -n "$GITHUB_ACTIONS" ] ; then + EDK_VARIANTS=$(for v in $INCLUDED_VARIANTS ; do + if [ -d variants/$v/llext-edk/include ] ; then echo $v ; fi + done) + if [ $(echo $EDK_VARIANTS | wc -w) -ge 2 ] ; then + extra/dedupe_includes.py --common-dir include $EDK_VARIANTS + echo "variants/include/" >> ${TEMP_INC} + fi +fi + # create the list of files and directories to exclude TEMP_EXC=$(mktemp -p . | sed 's/\.\///') declutter_file extra/artifacts/_common.exc >> ${TEMP_EXC} diff --git a/platform.txt b/platform.txt index bc8821cf4..de0b9387b 100644 --- a/platform.txt +++ b/platform.txt @@ -40,7 +40,7 @@ compiler.zephyr.includes_file={build.variant.path}/includes.txt compiler.zephyr.machine_flags_file={build.variant.path}/machine_flags.txt compiler.zephyr.cflags_file={build.variant.path}/cflags.txt compiler.zephyr.cxxflags_file={build.variant.path}/cxxflags.txt -compiler.zephyr.macros="-imacros{build.variant.path}/llext-edk/include/zephyr/include/generated/zephyr/autoconf.h" "-imacros{build.variant.path}/llext-edk/include/zephyr/include/zephyr/toolchain/zephyr_stdint.h" +compiler.zephyr.macros=-imacroszephyr/autoconf.h -imacroszephyr/toolchain/zephyr_stdint.h # NOTE: -fno-extern-tls-init prevents the C++ frontend from emitting guarded # references to the thread-local init function (e.g. _ZTH5errno) for extern # thread_local variables such as picolibc's `errno`. These variables are From bcf70cad5a616899b7ae94104f5ab9d937fdcc92 Mon Sep 17 00:00:00 2001 From: pennam Date: Fri, 16 Oct 2026 13:22:00 +0200 Subject: [PATCH 5/6] extra/dedupe_includes.py: normalize includes.txt paths Entries like '.../loader/ota/../../libraries/OTA/src' or '.../kernel_service/thread/../' were not normalized, since pathlib keeps '..' components. They never matched the deduplicated files, so no entry for the common directory was added and headers such as ota.h and kernel_shell.h could no longer be found (e.g. in zephyr_contrib). Normalizing the entries also makes the deepest-first sort use the real directory depth. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- extra/dedupe_includes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extra/dedupe_includes.py b/extra/dedupe_includes.py index 4b58f0db7..b2b449b0f 100755 --- a/extra/dedupe_includes.py +++ b/extra/dedupe_includes.py @@ -20,6 +20,7 @@ import argparse import hashlib import os +import posixpath import shutil import sys from pathlib import Path @@ -73,8 +74,10 @@ def update_includes_txt(name: str, common_dir: Path, common_rel_paths) -> None: return lines = includes_txt.read_text().splitlines() + # Normalize entries such as "a/b/../c" or "a/./b": pathlib keeps ".." + # components, so they would never match the deduplicated file paths. entries = [ - (i, line[len(INCLUDE_FLAG) :]) + (i, posixpath.normpath(line[len(INCLUDE_FLAG) :])) for i, line in enumerate(lines) if line.startswith(INCLUDE_FLAG) ] From 68223d14d8445dad33b3503d9edcf5ca50b5e886 Mon Sep 17 00:00:00 2001 From: pennam Date: Fri, 16 Oct 2026 13:36:57 +0200 Subject: [PATCH 6/6] package_core: handle variants/include outside of CI packaging The sync-zephyr-artifacts tool copies the deduplicated 'variants/include' folder from the downloaded core into the working tree: - ignore it in git, so it is not committed by mistake; - remove it in the sync tool before copying, so stale headers from a previous sync do not stay reachable through includes.txt; - add it to the archive in package_core.sh whenever it exists, so local packages built from a synced tree are not missing headers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 1 + extra/package_core.sh | 5 ++++- tools/sync-zephyr-artifacts/main.go | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8965cf832..5290a6187 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /platform.local.txt /loader/VERSION /cores/arduino/api/ +/variants/include/ __pycache__/ diff --git a/extra/package_core.sh b/extra/package_core.sh index 5000bf52e..867b565b7 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -96,9 +96,12 @@ if [ -n "$GITHUB_ACTIONS" ] ; then done) if [ $(echo $EDK_VARIANTS | wc -w) -ge 2 ] ; then extra/dedupe_includes.py --common-dir include $EDK_VARIANTS - echo "variants/include/" >> ${TEMP_INC} fi fi +# shared headers, from the step above or from a synced deduplicated package +if [ -d variants/include ] ; then + echo "variants/include/" >> ${TEMP_INC} +fi # create the list of files and directories to exclude TEMP_EXC=$(mktemp -p . | sed 's/\.\///') diff --git a/tools/sync-zephyr-artifacts/main.go b/tools/sync-zephyr-artifacts/main.go index 8fdf3b56e..209a68a7d 100644 --- a/tools/sync-zephyr-artifacts/main.go +++ b/tools/sync-zephyr-artifacts/main.go @@ -185,8 +185,9 @@ func main() { _ = cmd.Run() // no error check, the api symlink may be gone in the future } - // Remove old firmwares and variants/*/llext-edk files + // Remove old firmwares, variants/*/llext-edk files and shared variant headers os.RemoveAll(filepath.Join(gitCorePath, "firmwares")) + os.RemoveAll(filepath.Join(gitCorePath, "variants", "include")) filepath.WalkDir(filepath.Join(gitCorePath, "variants"), func(path string, d os.DirEntry, err error) error { if err != nil { fmt.Println("Error:", err)