diff --git a/.gitignore b/.gitignore index 5c63dce..f6d1e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,7 @@ _dummy_pxmodrim.py # OpenCode .opencode/** -opencode.json \ No newline at end of file +opencode.json + +# Steam workshop plugin +src/pxmodrim/ui/plugins/steam_workshop/ \ No newline at end of file diff --git a/companion-mods/rimworld-utils b/companion-mods/rimworld-utils index 60438f5..796809a 160000 --- a/companion-mods/rimworld-utils +++ b/companion-mods/rimworld-utils @@ -1 +1 @@ -Subproject commit 60438f539271420abe5ebcf1505abfeb0d84b65c +Subproject commit 796809a007be73abd1a5de19da59c321825faa48 diff --git a/justfile b/justfile index 445b0d4..4bff771 100644 --- a/justfile +++ b/justfile @@ -40,6 +40,21 @@ linux-copy-desktop: sed "s|Icon=pxmodrim|Icon={{justfile_directory()}}/src/pxmodrim/ui/assets/logo.svg|" packaging/linux/pxmodrim.desktop > ~/.local/share/applications/pxmodrim.desktop kbuildsycoca6 --noincremental +bench name="": + #!/usr/bin/env python3 + import glob, subprocess, sys + from pathlib import Path + root = Path("{{justfile_directory()}}") / "tests" / "benchmarks" + scripts = sorted(glob.glob(str(root / "bench_*.py"))) + name = "{{name}}" + if name and name != "all": + scripts = [s for s in scripts if name in s] + if not scripts: + print("no benchmark found"); sys.exit(1) + for s in scripts: + print(f" === {Path(s).stem} ===") + subprocess.run(["uv", "run", "python", s]) + build: uv run python packaging/build.py diff --git a/packaging/build.py b/packaging/build.py index 82aba30..8b2b014 100644 --- a/packaging/build.py +++ b/packaging/build.py @@ -16,6 +16,7 @@ def get_standalone_args(release: bool = False) -> list[str]: "nuitka", "--standalone", "--enable-plugin=pyside6", + "--nofollow-import-to=pytest,pygments", "--include-package-data=pxmodrim", f"--output-dir={project_root / 'dist'}", "--assume-yes-for-downloads", diff --git a/pyproject.toml b/pyproject.toml index 39ba7b6..375c43f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,7 @@ dependencies = [ "loguru>=0.7.3", "lxml>=6.1.1", "msgspec>=0.21.1", - "natsort>=8.4.0", - "networkx>=3.6.1", - "pyside6>=6.11.1,<6.12", + "pyside6>=6.11.1", "qasync>=0.28.0", "toposort>=1.10", "ttimer>=0.0.2", diff --git a/src/pxmodrim/core/checker/sort.py b/src/pxmodrim/core/checker/sort.py index 8bfde5b..e9fc0b5 100644 --- a/src/pxmodrim/core/checker/sort.py +++ b/src/pxmodrim/core/checker/sort.py @@ -1,5 +1,6 @@ from __future__ import annotations +import heapq from typing import TYPE_CHECKING from pxmodrim.core.checker.graph import ConstraintGraph, EdgeType @@ -62,17 +63,16 @@ def topological_sort( def _key(pid: PackageId, config_priority=config_priority) -> tuple: return (config_priority.get(pid, len(config_priority)), pid.lower()) - queue = [pid for pid, deg in indegree.items() if deg == 0] - queue.sort(key=_key) + queue = [(_key(pid), pid) for pid, deg in indegree.items() if deg == 0] + heapq.heapify(queue) while queue: - pid = queue.pop(0) + _, pid = heapq.heappop(queue) order.append(pid) for dependent in rev.get(pid, set()): indegree[dependent] -= 1 if indegree[dependent] == 0: - queue.append(dependent) - queue.sort(key=_key) + heapq.heappush(queue, (_key(dependent), dependent)) remaining = tier_pids - set(order) if remaining: @@ -93,20 +93,18 @@ def _build_deps_and_rev_deps( if pid not in all_pids: continue - for edge in graph.edges_of_type(pid, EdgeType.LOAD_BEFORE): - if edge.target in all_pids: - deps.setdefault(edge.target, set()).add(pid) - rev_deps.setdefault(pid, set()).add(edge.target) - - for edge in graph.edges_of_type(pid, EdgeType.LOAD_AFTER): - if edge.target in all_pids: - deps.setdefault(pid, set()).add(edge.target) - rev_deps.setdefault(edge.target, set()).add(pid) - - for edge in graph.edges_of_type(pid, EdgeType.DEPENDENCY): - if edge.target in all_pids: - deps.setdefault(pid, set()).add(edge.target) - rev_deps.setdefault(edge.target, set()).add(pid) + deps_pid = deps[pid] + rev_pid = rev_deps[pid] + for edge in graph.outgoing(pid): + target = edge.target + if target not in all_pids: + continue + if edge.type == EdgeType.LOAD_BEFORE: + deps[target].add(pid) + rev_pid.add(target) + elif edge.type in (EdgeType.LOAD_AFTER, EdgeType.DEPENDENCY): + deps_pid.add(target) + rev_deps[target].add(pid) return deps, rev_deps diff --git a/src/pxmodrim/core/models/metadata/parsing.py b/src/pxmodrim/core/models/metadata/parsing.py index 2ef2f48..ebd1af0 100644 --- a/src/pxmodrim/core/models/metadata/parsing.py +++ b/src/pxmodrim/core/models/metadata/parsing.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +import lxml.etree as ET from loguru import logger from pxmodrim.core.constants import ( @@ -22,7 +23,7 @@ ListedMod, ) from pxmodrim.core.utils import find_about_xml -from pxmodrim.core.xml import xml_path_to_json +from pxmodrim.core.xml import _text class MalformedDataException(Exception): @@ -424,6 +425,264 @@ def create_about_mod( return mod.valid, mod +def _match_versioned_child( + parent: ET._Element, target_version: str +) -> ET._Element | None: + """Find the versioned child element matching target_version (*.*).""" + try: + major, minor = target_version.split(".")[:2] + except ValueError: + return None + key = f"v{major}.{minor}" + found = parent.find(key) + if found is None: + found = parent.find(f"{major}.{minor}") + if found is not None: + return found + for child in parent: + if re.match(key, child.tag): + return child + return None + + +def _dep_from_li(li: ET._Element) -> DependencyMod: + """Parse a
  • element from into a DependencyMod.""" + dep = DependencyMod() + pid = _text(li, "packageId") + if pid: + dep.package_id = CaseInsensitiveStr(pid) + name = _text(li, "displayName") + if name: + dep.name = name + url = _text(li, "workshopUrl") + if url: + dep.workshop_url = url + alt_el = li.find("alternativePackageIds") + if alt_el is not None: + for alt_li in alt_el: + if alt_li.tag == "li" and alt_li.text and alt_li.text.strip(): + dep.alternative_package_ids.add( + CaseInsensitiveStr(alt_li.text.strip()) + ) + return dep + + +def _element_value(el: ET._Element | None) -> str | list[str] | None: + """Extract text or
  • children as a string or list of strings.""" + if el is None: + return None + children = [ch for ch in el if ch.tag == "li"] + if children: + return [ + c.text.strip() + for c in children + if c.text and c.text.strip() + ] + if el.text: + text = el.text.strip() + return text or None + return None + + +def _create_about_mod_from_element( + root: ET._Element, + target_version: str, + prefer_versioned: bool = True, +) -> AboutXmlMod: + """Build AboutXmlMod in a single pass over the element children.""" + mod = AboutXmlMod() + rules = BaseRules() + + deps_el: ET._Element | None = None + deps_bv: bool = False + load_before_li: list[str] = [] + load_after_li: list[str] = [] + incompat_li: list[str] = [] + force_before_li: list[str] = [] + force_after_li: list[str] = [] + load_before_bv_el: ET._Element | None = None + load_after_bv_el: ET._Element | None = None + incompat_bv_el: ET._Element | None = None + + for child in root: + tag = child.tag + if tag == "packageId": + t = _element_value(child) + if isinstance(t, str) and t: + mod.package_id = CaseInsensitiveStr(t) + else: + _set_mod_invalid( + mod, + f"packageId missing or invalid: {t}. " + f"Assigned sentinel '{DEFAULT_MISSING_PACKAGEID}'.", + ) + mod.package_id = CaseInsensitiveStr(DEFAULT_MISSING_PACKAGEID) + + elif tag == "steamAppId": + t = _element_value(child) + if isinstance(t, str) and t.isdigit(): + mod.steam_app_id = int(t) + + elif tag == "name": + t = _element_value(child) + if isinstance(t, str): + mod.name = t + + elif tag == "description": + t = _element_value(child) + if isinstance(t, str): + mod.description = t + + elif tag == "author": + t = _element_value(child) + if isinstance(t, str): + mod.authors.append(t) + + elif tag == "authors": + t = _element_value(child) + if isinstance(t, list): + mod.authors.extend(t) + + elif tag == "supportedVersions": + t = _element_value(child) + if isinstance(t, list): + mod.supported_versions = set(t) + + elif tag == "modVersion": + t = _element_value(child) + if isinstance(t, str): + mod.mod_version = t + + elif tag == "modIconPath": + t = _element_value(child) + if isinstance(t, str): + mod.mod_icon_path = Path(t) + + elif tag == "url": + t = _element_value(child) + if isinstance(t, str): + mod.url = t + + elif tag == "modDependencies": + if not deps_bv: + deps_el = child + elif tag == "modDependenciesByVersion": + if prefer_versioned: + matched = _match_versioned_child(child, target_version) + if matched is not None: + deps_el = matched + deps_bv = True + + elif tag == "loadBefore": + t = _element_value(child) + if isinstance(t, list): + load_before_li = t + elif tag == "loadBeforeByVersion": + if prefer_versioned: + matched = _match_versioned_child(child, target_version) + if matched is not None: + load_before_bv_el = matched + + elif tag == "loadAfter": + t = _element_value(child) + if isinstance(t, list): + load_after_li = t + elif tag == "loadAfterByVersion": + if prefer_versioned: + matched = _match_versioned_child(child, target_version) + if matched is not None: + load_after_bv_el = matched + + elif tag == "forceLoadBefore": + t = _element_value(child) + if isinstance(t, list): + force_before_li = t + elif tag == "forceLoadAfter": + t = _element_value(child) + if isinstance(t, list): + force_after_li = t + + elif tag == "incompatibleWith": + t = _element_value(child) + if isinstance(t, list): + incompat_li = t + elif tag == "incompatibleWithByVersion": + if prefer_versioned: + matched = _match_versioned_child(child, target_version) + if matched is not None: + incompat_bv_el = matched + + elif tag == "descriptionsByVersion": + matched = _match_versioned_child(child, target_version) + if matched is not None and matched.text and matched.text.strip(): + mod.description = matched.text.strip() + + # Apply versioned overrides + if load_before_bv_el is not None: + t = _element_value(load_before_bv_el) + if isinstance(t, list): + load_before_li = t + if load_after_bv_el is not None: + t = _element_value(load_after_bv_el) + if isinstance(t, list): + load_after_li = t + if incompat_bv_el is not None: + t = _element_value(incompat_bv_el) + if isinstance(t, list): + incompat_li = t + + # Combine force + regular + load_before_li.extend(force_before_li) + load_after_li.extend(force_after_li) + + # Build rules + rules.load_before = CaseInsensitiveSet(load_before_li) + rules.load_after = CaseInsensitiveSet(load_after_li) + rules.incompatible_with = CaseInsensitiveSet(incompat_li) + + # Process dependencies + if deps_el is not None: + for li in deps_el: + if li.tag != "li": + continue + if li.attrib.get("isNull") == "True": + continue + if not li.attrib and len(li) == 0: + continue + dep = _dep_from_li(li) + if dep.package_id in rules.dependencies: + logger.warning( + f"Duplicate dependency found: {dep.package_id}. Skipping." + ) + else: + rules.dependencies[dep.package_id] = dep + + mod.about_rules = rules + + # DLC fallback for name/description/steamAppId + str_pid = str(mod.package_id) + dlc_appid = _get_dlc_packageid_map().get(str_pid) + dlc_meta = RIMWORLD_DLC_METADATA.get(dlc_appid, {}) if dlc_appid else {} + if dlc_meta: + if not mod.name or mod.name == str_pid: + mod.name = dlc_meta["name"] + if not mod.description: + mod.description = dlc_meta["description"] + + # DLC -> RimWorld dependency + dlc_map = _get_dlc_packageid_map() + if str_pid in dlc_map and dlc_map[str_pid] != RIMWORLD_STEAM_APP_ID: + rimworld_pid = CaseInsensitiveStr("ludeon.rimworld") + if rimworld_pid not in rules.dependencies: + rules.dependencies[rimworld_pid] = DependencyMod( + package_id=rimworld_pid, + name="RimWorld", + workshop_url="https://store.steampowered.com/app/294100/RimWorld", + ) + + return mod + + def _create_about_mod_from_xml( base_path: Path, mod_xml_path: Path, @@ -432,38 +691,31 @@ def _create_about_mod_from_xml( ) -> tuple[bool, AboutXmlMod]: """Parse an About.xml file and return a validated AboutXmlMod with its path set.""" try: - mod_data = xml_path_to_json(str(mod_xml_path)) + tree = ET.parse(str(mod_xml_path)) except (OSError, TypeError): logger.error(f"Unable to parse {mod_xml_path}: {traceback.format_exc()}") return False, AboutXmlMod(valid=False) - mod_data = {k.lower(): v for k, v in mod_data.items()} - mod_data = mod_data.get("modmetadata", {}) - - if not mod_data: + root = tree.getroot() + if root is None: logger.error(f"Could not parse {mod_xml_path}.") return False, AboutXmlMod(valid=False) - valid, mod = create_about_mod(mod_data, target_version, prefer_versioned) - + mod = _create_about_mod_from_element(root, target_version, prefer_versioned) mod.mod_path = base_path - return valid, mod + return mod.valid, mod def create_listed_mod_from_path( path: Path, target_version: str, prefer_versioned: bool = True, - case_insensitive_about_xml: bool = True, + about_xml_path: Path | None = None, ) -> tuple[bool, ListedMod]: """Create a ListedMod from a directory path, parsing About.xml if present.""" if path.is_dir(): - about_xml_path: Path | None - if case_insensitive_about_xml: + if about_xml_path is None: about_xml_path = find_about_xml(path) - else: - candidate = path / "About" / "About.xml" - about_xml_path = candidate if candidate.exists() else None if about_xml_path is not None: success, about_mod = _create_about_mod_from_xml( diff --git a/src/pxmodrim/core/providers/core.py b/src/pxmodrim/core/providers/core.py index 2192ab9..1c80a77 100644 --- a/src/pxmodrim/core/providers/core.py +++ b/src/pxmodrim/core/providers/core.py @@ -37,9 +37,11 @@ def _scan(t: Timer | None) -> dict[str, ListedMod]: if data_dir.exists(): with (t or Timer())("scan_dir"): dirs = scan_mod_directory(data_dir) - for d in dirs: + for d, a in dirs.items(): with (t or Timer())("parse_xml"): - _, mod = create_listed_mod_from_path(d, target_version) + _, mod = create_listed_mod_from_path( + d, target_version, about_xml_path=a + ) logger.debug( "CoreModProvider found: {} (uuid: {})", mod.name, mod.uuid ) diff --git a/src/pxmodrim/core/providers/local.py b/src/pxmodrim/core/providers/local.py index 09e38f9..6cfce76 100644 --- a/src/pxmodrim/core/providers/local.py +++ b/src/pxmodrim/core/providers/local.py @@ -38,17 +38,19 @@ def _scan(t: Timer | None) -> dict[str, ListedMod]: with tm("scan_dir"): dirs = scan_mod_directory(self._path) - def _keep(d: Path) -> ListedMod | None: + def _keep(d: Path, about: Path) -> ListedMod | None: if (d / "About/PublishedFileId.txt").exists(): return None - _, mod = create_listed_mod_from_path(d, target_version) + _, mod = create_listed_mod_from_path( + d, target_version, about_xml_path=about + ) mod.provider_id = self.provider_id return mod pool = self._pool if pool is not None: with tm("process_mods"): - futures = {pool.submit(_keep, d): d for d in dirs} + futures = {pool.submit(_keep, d, a): d for d, a in dirs.items()} result: dict[str, ListedMod] = {} for f in as_completed(futures): mod = f.result() @@ -57,8 +59,8 @@ def _keep(d: Path) -> ListedMod | None: else: with tm("process_mods"): result = {} - for d in dirs: - mod = _keep(d) + for d, a in dirs.items(): + mod = _keep(d, a) if mod is not None: result[mod.uuid] = mod return result @@ -94,17 +96,19 @@ def _scan(t: Timer | None) -> dict[str, ListedMod]: with tm("scan_dir"): dirs = scan_mod_directory(self._path) - def _keep(d: Path) -> ListedMod | None: + def _keep(d: Path, about: Path) -> ListedMod | None: if not (d / "About/PublishedFileId.txt").exists(): return None - _, mod = create_listed_mod_from_path(d, target_version) + _, mod = create_listed_mod_from_path( + d, target_version, about_xml_path=about + ) mod.provider_id = self.provider_id return mod pool = self._pool if pool is not None: with tm("process_mods"): - futures = {pool.submit(_keep, d): d for d in dirs} + futures = {pool.submit(_keep, d, a): d for d, a in dirs.items()} result: dict[str, ListedMod] = {} for f in as_completed(futures): mod = f.result() @@ -113,8 +117,8 @@ def _keep(d: Path) -> ListedMod | None: else: with tm("process_mods"): result = {} - for d in dirs: - mod = _keep(d) + for d, a in dirs.items(): + mod = _keep(d, a) if mod is not None: result[mod.uuid] = mod return result diff --git a/src/pxmodrim/core/services/mod_discovery.py b/src/pxmodrim/core/services/mod_discovery.py index 2300574..bdc189f 100644 --- a/src/pxmodrim/core/services/mod_discovery.py +++ b/src/pxmodrim/core/services/mod_discovery.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path from loguru import logger @@ -9,18 +10,21 @@ from pxmodrim.core.utils import find_about_xml -def scan_mod_directory(mods_path: Path) -> list[Path]: - """Scan a directory for mod subdirectories, returning those containing About.xml.""" +def scan_mod_directory(mods_path: Path) -> dict[Path, Path]: + """Scan a directory for mod subdirectories, returning {mod_path: about_xml_path}.""" if not mods_path.exists() or not mods_path.is_dir(): logger.warning(f"Mod directory not found: {mods_path}") - return [] - - results: list[Path] = [] - for entry in mods_path.iterdir(): - if not entry.is_dir(): - continue - if find_about_xml(entry): - results.append(entry) + return {} + + results: dict[Path, Path] = {} + with os.scandir(mods_path) as it: + for entry in it: + if not entry.is_dir(): + continue + p = Path(entry.path) + about = find_about_xml(p) + if about: + results[p] = about return results diff --git a/src/pxmodrim/core/utils.py b/src/pxmodrim/core/utils.py index 704136b..b18f713 100644 --- a/src/pxmodrim/core/utils.py +++ b/src/pxmodrim/core/utils.py @@ -7,6 +7,9 @@ def find_about_xml(mod_path: Path) -> Path | None: """Find About.xml in a mod directory (case-insensitive).""" if not mod_path.is_dir(): return None + candidate = mod_path / "About" / "About.xml" + if candidate.is_file(): + return candidate for entry in mod_path.iterdir(): if entry.name.lower() == "about" and entry.is_dir(): for child in entry.iterdir(): diff --git a/src/pxmodrim/core/xml.py b/src/pxmodrim/core/xml.py index 6d092ef..dfffae8 100644 --- a/src/pxmodrim/core/xml.py +++ b/src/pxmodrim/core/xml.py @@ -7,6 +7,15 @@ from loguru import logger +def _text(elem: ET._Element, tag: str) -> str | None: + """Return stripped text of the first child matching *tag*, or None.""" + found = elem.find(tag) + if found is None or found.text is None: + return None + text = found.text.strip() + return text or None + + def etree_to_dict(t: Any) -> dict[str, Any]: """Recursively convert an lxml Element tree into a plain nested dictionary.""" d: dict[str, Any] = {str(t.tag): {}} diff --git a/src/pxmodrim/ui/panels/about_panel.py b/src/pxmodrim/ui/panels/about_panel.py index 9379ab1..bc697ab 100644 --- a/src/pxmodrim/ui/panels/about_panel.py +++ b/src/pxmodrim/ui/panels/about_panel.py @@ -139,8 +139,6 @@ def _create_credits_tab(self) -> QWidget: ("loguru", "MIT"), ("lxml", "BSD-3-Clause"), ("msgspec", "BSD-3-Clause"), - ("natsort", "MIT"), - ("networkx", "BSD-3-Clause"), ("pyside6", "LGPL-3.0 / GPL-2.0 / GPL-3.0"), ("qasync", "BSD-2-Clause"), ("toposort", "Apache-2.0"), diff --git a/src/pxmodrim/ui/window/main_window.py b/src/pxmodrim/ui/window/main_window.py index dbf45e9..f72d22b 100644 --- a/src/pxmodrim/ui/window/main_window.py +++ b/src/pxmodrim/ui/window/main_window.py @@ -343,9 +343,14 @@ async def _auto_sort(self) -> None: elapsed = time.monotonic() - t0 self.mod_list.model.reorder(ordered_uuids) self._ctx.diagnostics_service.reorder(ordered_uuids) - logger.info("auto-sort: {} mods sorted in {:.1f}s", len(ordered_uuids), elapsed) + elapsed_ms = elapsed * 1000 + logger.info( + "auto-sort: {} mods sorted in {:.0f}ms", + len(ordered_uuids), + elapsed_ms, + ) self._toast_manager.success( - f"Sorted {len(ordered_uuids)} mods in {elapsed:.1f}s", 5000 + f"Sorted {len(ordered_uuids)} mods in {elapsed_ms:.0f}ms", 5000 ) @asyncSlot() diff --git a/tests/benchmarks/bench_discovery.py b/tests/benchmarks/bench_discovery.py new file mode 100644 index 0000000..95ac483 --- /dev/null +++ b/tests/benchmarks/bench_discovery.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import random +import string +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from loguru import logger + +logger.remove() + +from pxmodrim.core.models.metadata.parsing import ( # noqa: E402 + create_listed_mod_from_path, +) +from pxmodrim.core.services.mod_discovery import scan_mod_directory # noqa: E402 + +ABOUT_XML = """\ + + + {name} + {author} + {author}.{name} + https://steamcommunity.com/sharedfiles/filedetails/?id={sid} + +
  • 1.4
    • +
  • 1.5
    • + + +
  • {dep}
    • +     + A benchmark mod #{i} + 1.0.0 +""" + + +def make_mod_dir(root: Path, i: int, dep_pid: str) -> Path: + author = random.choice(string.ascii_lowercase[:5]) + name = f"benchmark_mod_{i:04d}" + + about = ABOUT_XML.format( + name=name, author=author, sid=i, dep=dep_pid, i=i + ) + + mod_dir = root / name + about_dir = mod_dir / "About" + about_dir.mkdir(parents=True, exist_ok=True) + (about_dir / "About.xml").write_text(about) + (about_dir / "PublishedFileId.txt").write_text(str(i)) + return mod_dir + + +def make_mods(root: Path, count: int) -> None: + """Create count mod directories with dependency chains.""" + prev_pid = "ludeon.rimworld" + for i in range(count): + make_mod_dir(root, i, prev_pid) + author = random.choice(string.ascii_lowercase[:5]) + prev_pid = f"{author}.benchmark_mod_{i:04d}" + + +def bench(count: int) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "Mods" + root.mkdir() + make_mods(root, count) + + t0 = time.perf_counter_ns() + scanned = scan_mod_directory(root) + t1 = time.perf_counter_ns() + + mods = {} + t2 = time.perf_counter_ns() + for d, about in scanned.items(): + _, mod = create_listed_mod_from_path( + d, "1.5", about_xml_path=about + ) + mods[mod.uuid] = mod + t3 = time.perf_counter_ns() + + scan_ms = (t1 - t0) / 1e6 + parse_ms = (t3 - t2) / 1e6 + total_ms = (t3 - t0) / 1e6 + print( + f"{count:>5} mods: scan_dir={scan_ms:>6.1f}ms" + f" parse_xml={parse_ms:>6.1f}ms total={total_ms:>6.1f}ms" + ) + print( + f" scan avg: {scan_ms / count * 1000:.1f} us/mod" + f" parse avg: {parse_ms / count * 1000:.1f} us/mod" + ) + + +if __name__ == "__main__": + for count in (50, 100, 200, 500, 1000, 2000): + bench(count) diff --git a/tests/benchmarks/bench_sort.py b/tests/benchmarks/bench_sort.py new file mode 100644 index 0000000..6d1d785 --- /dev/null +++ b/tests/benchmarks/bench_sort.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import random +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from loguru import logger + +logger.remove() + +from pxmodrim.core.checker.graph import ConstraintGraph # noqa: E402 +from pxmodrim.core.checker.sort import topological_sort # noqa: E402 +from pxmodrim.core.models.metadata.structures import ( # noqa: E402 + AboutXmlMod, + BaseRules, + CaseInsensitiveSet, + CaseInsensitiveStr, + DependencyMod, +) +from pxmodrim.core.sort.config import SortSettings, TierConfig # noqa: E402 +from pxmodrim.core.sort.models import CommunityRule # noqa: E402 + +PackageId = CaseInsensitiveStr + + +def _tier_config(pids: list[PackageId]) -> TierConfig: + return TierConfig( + tier_0=tuple(pids[:3]) if len(pids) >= 3 else tuple(pids), + tier_1=tuple(pids[3:6]) if len(pids) >= 6 else (), + tier_3=(), + ) + + +def make_mods( + count: int, edges_per_mod: int +) -> tuple[ + dict[PackageId, AboutXmlMod], + dict[PackageId, CommunityRule], +]: + rng = random.Random(42) + pids = [PackageId(f"bench.mod.{i:04d}") for i in range(count)] + community_rules: dict[PackageId, CommunityRule] = {} + mods: dict[PackageId, AboutXmlMod] = {} + + for i, pid in enumerate(pids): + deps: dict[PackageId, DependencyMod] = {} + load_after = CaseInsensitiveSet() + + num_edges = min(edges_per_mod, i) + if num_edges: + targets = rng.sample(range(i), k=num_edges) + dep_count = max(1, num_edges // 2) + for idx in targets[:dep_count]: + deps[pids[idx]] = DependencyMod( + name=str(pids[idx]), + package_id=pids[idx], + ) + for idx in targets[dep_count:]: + load_after.add(pids[idx]) + + mods[pid] = AboutXmlMod( + name=f"Bench Mod {i:04d}", + package_id=pid, + provider_id="bench", + valid=True, + about_rules=BaseRules(dependencies=deps, load_after=load_after), + ) + + if rng.random() < 0.2: + cr_la: set[PackageId] = set() + if i > 0 and rng.random() < 0.5: + extra = rng.randint(1, min(2, i)) + for idx in rng.sample(range(i), k=extra): + cr_la.add(pids[idx]) + + community_rules[pid] = CommunityRule( + package_id=pid, + load_after=cr_la, + load_before=set(), + load_first=rng.random() < 0.1, + load_last=rng.random() < 0.1, + incompatible_with=set(), + ) + + return mods, community_rules + + +def bench(count: int, edges_per_mod: int) -> None: + mods, community_rules = make_mods(count, edges_per_mod) + pids = list(mods.keys()) + settings = SortSettings( + use_community_rules=True, + use_alternative_package_ids=False, + tier_config=_tier_config(pids), + ) + + graph = ConstraintGraph() + + t0 = time.perf_counter_ns() + graph.build(mods, pids, settings, community_rules) + t1 = time.perf_counter_ns() + + t2 = time.perf_counter_ns() + result = topological_sort(mods, graph, settings, community_rules) + t3 = time.perf_counter_ns() + + build_ms = (t1 - t0) / 1e6 + sort_ms = (t3 - t2) / 1e6 + total_ms = (t3 - t0) / 1e6 + + assert len(result) == count + + print( + f"{count:>5} mods ({edges_per_mod} e/m): " + f"graph={build_ms:>6.1f}ms " + f"sort={sort_ms:>6.1f}ms " + f"total={total_ms:>6.1f}ms" + ) + print( + f" avg per mod: " + f"graph={build_ms/count*1000:>5.1f}us " + f"sort={sort_ms/count*1000:>5.1f}us " + f"total={total_ms/count*1000:>5.1f}us" + ) + + +if __name__ == "__main__": + print("=== bench_sort ===") + for count in (200, 500, 1000, 2000): + bench(count, edges_per_mod=5) + bench(2000, edges_per_mod=10) diff --git a/uv.lock b/uv.lock index d01869e..113f2ef 100644 --- a/uv.lock +++ b/uv.lock @@ -171,24 +171,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345, upload-time = "2026-04-12T21:44:16.431Z" }, ] -[[package]] -name = "natsort" -version = "8.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/a9/a0c57aee75f77794adaf35322f8b6404cbd0f89ad45c87197a937764b7d0/natsort-8.4.0.tar.gz", hash = "sha256:45312c4a0e5507593da193dedd04abb1469253b601ecaf63445ad80f0a1ea581", size = 76575, upload-time = "2023-06-20T04:17:19.925Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c", size = 38268, upload-time = "2023-06-20T04:17:17.522Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - [[package]] name = "nodeenv" version = "1.10.0" @@ -232,8 +214,6 @@ dependencies = [ { name = "loguru", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "lxml", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "msgspec", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "natsort", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "networkx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pyside6", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "qasync", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "toposort", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -259,9 +239,7 @@ requires-dist = [ { name = "loguru", specifier = ">=0.7.3" }, { name = "lxml", specifier = ">=6.1.1" }, { name = "msgspec", specifier = ">=0.21.1" }, - { name = "natsort", specifier = ">=8.4.0" }, - { name = "networkx", specifier = ">=3.6.1" }, - { name = "pyside6", specifier = ">=6.11.1,<6.12" }, + { name = "pyside6", specifier = ">=6.11.1" }, { name = "qasync", specifier = ">=0.28.0" }, { name = "toposort", specifier = ">=1.10" }, { name = "ttimer", specifier = ">=0.0.2" },