diff --git a/docs/source/package_orderers.rst b/docs/source/package_orderers.rst index d7b56e7b98..0f217bf076 100644 --- a/docs/source/package_orderers.rst +++ b/docs/source/package_orderers.rst @@ -14,6 +14,35 @@ Configuration Package orderers can be configured in the ``rezconfig.py`` via the :data:`package_orderers` setting. +Environment Variable Configuration +================================== + +Package orderers can also be configured at runtime using the +``REZ_PACKAGE_ORDERERS_JSON`` environment variable. The value must be a +JSON-encoded list of orderer definitions, using the same structure as the +:data:`package_orderers` setting: + +.. code-block:: bash + + export REZ_PACKAGE_ORDERERS_JSON='[{"type": "per_family", "orderers": [{"packages": ["python"], "type": "version_split", "first_version": "2.7.16"}]}]' + +This is useful for configuring orderers with constructor arguments (such as +prerelease risk tolerance or custom version priority ranges) without modifying +``rezconfig.py``. + +.. note:: + + The environment variable is read on first access to the configuration. + If you need to change it at runtime via the API, clear both caches: + + .. code-block:: python + + from rez.config import config + from rez.package_order import PackageOrderList + + config._uncache("package_orderers") + PackageOrderList.clear_singleton_cache() + Types ===== @@ -188,9 +217,20 @@ package. You would use a :class:`rez.package_order.NullPackageOrder` in a :class Custom orderers =============== -It is possible to create custom orderers using the API. This can be achieved -by subclassing :class:`rez.package_order.PackageOrder` and implementing some mandatory -methods. Once that's done, you need to register the orderer using :func:`rez.package_order.register_orderer`. +It is possible to create custom orderers using the API. The preferred approach +is to create a plugin in ``rezplugins/package_order/``. Alternatively, you can +subclass :class:`rez.package_order.PackageOrder` and register the orderer using +:func:`rez.package_order.register_orderer` (API-based registration). + +.. note:: + + The plugin system is preferred for persistent, facility-wide orderers. + Use ``register_orderer()`` for dynamic or programmatic registration. + +.. seealso:: + + For plugin-based orderer examples, see the built-in orderers in + ``src/rezplugins/package_order/``. .. note:: diff --git a/src/rez/package_order.py b/src/rez/package_order.py index d8c7555753..7e21c687c1 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -9,17 +9,13 @@ from typing import Any, Callable, Iterable, List, TYPE_CHECKING from rez.config import config +from rez.exceptions import RezPluginError from rez.utils.data_utils import cached_class_property from rez.version import Version, VersionRange -from rez.version._version import _Comparable, _ReversedComparable, _LowerBound, _UpperBound, _Bound -from rez.packages import iter_packages, Package +from rez.version._version import _Comparable, _LowerBound, _UpperBound, _Bound +from rez.packages import Package from rez.utils.typing import SupportsLessThan -if TYPE_CHECKING: - # this is not available in typing until 3.11, but due to __future__.annotations - # we can use it without really importing it - from typing import Self - ALL_PACKAGES = "*" @@ -28,9 +24,7 @@ class FallbackComparable(_Comparable): fails, compares using the fallback_comparable object. """ - def __init__(self, - main_comparable: SupportsLessThan, - fallback_comparable: SupportsLessThan) -> None: + def __init__(self, main_comparable: SupportsLessThan, fallback_comparable: SupportsLessThan) -> None: self.main_comparable = main_comparable self.fallback_comparable = fallback_comparable @@ -47,7 +41,7 @@ def __lt__(self, other: object) -> bool: return self.fallback_comparable < other.fallback_comparable def __repr__(self) -> str: - return '%s(%r, %r)' % (type(self).__name__, self.main_comparable, self.fallback_comparable) + return "%s(%r, %r)" % (type(self).__name__, self.main_comparable, self.fallback_comparable) class PackageOrder(object): @@ -84,8 +78,7 @@ def packages(self, packages: str | Iterable[str] | None) -> None: else: self._packages = sorted(packages) - def reorder(self, iterable: Iterable[Package], - key: Callable[[Any], Package] | None = None) -> list[Package] | None: + def reorder(self, iterable: Iterable[Package], key: Callable[[Any], Package] | None = None) -> list[Package] | None: """Put packages into some order for consumption. You can safely assume that the packages referred to by `iterable` are @@ -108,14 +101,12 @@ def reorder(self, iterable: Iterable[Package], """ key = key or (lambda x: x) package_name = self._get_package_name_from_iterable(iterable, key=key) - return sorted(iterable, - key=lambda x: self.sort_key(package_name, key(x).version), - reverse=True) + return sorted(iterable, key=lambda x: self.sort_key(package_name, key(x).version), reverse=True) @staticmethod - def _get_package_name_from_iterable(iterable: Iterable[Package], - key: Callable[[Any], Package] | None = None - ) -> str | None: + def _get_package_name_from_iterable( + iterable: Iterable[Package], key: Callable[[Any], Package] | None = None + ) -> str | None: """Utility method for getting a package from an iterable""" try: item = next(iter(iterable)) @@ -125,9 +116,9 @@ def _get_package_name_from_iterable(iterable: Iterable[Package], key = key or (lambda x: x) return key(item).name - def sort_key(self, package_name: str, - version_like: Version | _LowerBound | _UpperBound | _Bound | VersionRange | None - ) -> SupportsLessThan: + def sort_key( + self, package_name: str, version_like: Version | _LowerBound | _UpperBound | _Bound | VersionRange | None + ) -> SupportsLessThan: """Returns a sort key usable for sorting packages within the same family Args: @@ -144,8 +135,7 @@ def sort_key(self, package_name: str, if isinstance(version_like, VersionRange): return tuple(self.sort_key(package_name, bound) for bound in version_like.bounds) if isinstance(version_like, _Bound): - return (self.sort_key(package_name, version_like.lower), - self.sort_key(package_name, version_like.upper)) + return (self.sort_key(package_name, version_like.lower), self.sort_key(package_name, version_like.upper)) if isinstance(version_like, _LowerBound): inclusion_key = -2 if version_like.inclusive else -1 return self.sort_key(package_name, version_like.version), inclusion_key @@ -154,8 +144,7 @@ def sort_key(self, package_name: str, return self.sort_key(package_name, version_like.version), inclusion_key if isinstance(version_like, Version): # finally, the bit that we actually use the sort_key_implementation for. - return FallbackComparable( - self.sort_key_implementation(package_name, version_like), version_like) + return FallbackComparable(self.sort_key_implementation(package_name, version_like), version_like) if version_like is None: # As no version range is provided for this package, # Python's sort preserves the order of equal elements. @@ -187,13 +176,13 @@ def from_pod(cls, data: dict[str, Any]) -> PackageOrder: @property def sha1(self) -> str: - return sha1(repr(self).encode('utf-8')).hexdigest() + return sha1(repr(self).encode("utf-8")).hexdigest() def __str__(self) -> str: raise NotImplementedError def __eq__(self, other): - return type(self) == type(other) and str(self) == str(other) # noqa: E721 + return type(self) is type(other) and str(self) == str(other) def __ne__(self, other) -> bool: return not self == other @@ -202,439 +191,55 @@ def __repr__(self) -> str: return "%s(%s)" % (self.__class__.__name__, str(self)) -class NullPackageOrder(PackageOrder): - """An orderer that does not change the order - a no op. - - This orderer is useful in cases where you want to apply some default orderer - to a set of packages, but may want to explicitly NOT reorder a particular - package. You would use a :class:`NullPackageOrder` in a :class:`PerFamilyOrder` to do this. - """ - name = "no_order" - - def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: - # python's sort will preserve the order of items that compare equal, so - # to not change anything, we just return the same object for all... - return 0 - - def __str__(self) -> str: - return "{}" - - def __eq__(self, other): - return type(self) == type(other) # noqa: E721 - - def to_pod(self) -> dict[str, Any]: - """ - Example (in yaml): - - .. code-block:: yaml - - type: no_order - packages: ["foo"] - """ - return { - "packages": self.packages, - } - - @classmethod - def from_pod(cls, data: dict[str, Any]) -> Self: - return cls(packages=data.get("packages")) - - -class SortedOrder(PackageOrder): - """An orderer that sorts based on :attr:`Package.version `. - """ - name = "sorted" - - def __init__(self, descending: bool, packages: list[str] | None = None) -> None: - super().__init__(packages) - self.descending = descending - - def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: - # Note that the name "descending" can be slightly confusing - it - # indicates that the final ordering this Order gives should be - # version descending (ie, the default) - however, the sort_key itself - # returns its results in "normal" ascending order (because it needs to - # be used "alongside" normally-sorted objects like versions). - # when the key is passed to sort(), though, it is always invoked with - # reverse=True... - if self.descending: - return version - else: - return _ReversedComparable(version) - - def __str__(self) -> str: - return str(self.descending) - - def __eq__(self, other): - return ( # noqa: E721 - type(self) == type(other) - and self.descending == other.descending - ) - - def to_pod(self) -> dict[str, Any]: - """ - Example (in yaml): - - .. code-block:: yaml - - type: sorted - descending: true - packages: ["foo"] - """ - return { - "descending": self.descending, - "packages": self.packages, - } - - @classmethod - def from_pod(cls, data: dict[str, Any]) -> Self: - return cls( - data["descending"], - packages=data.get("packages"), - ) - - -class PerFamilyOrder(PackageOrder): - """An orderer that applies different orderers to different package families. - """ - name = "per_family" - - def __init__(self, order_dict: dict[str, PackageOrder], - default_order: PackageOrder | None = None) -> None: - """Create a reorderer. - - Args: - order_dict (dict[str, PackageOrder]): Orderers to apply to - each package family. - default_order (PackageOrder): Orderer to apply to any packages - not specified in ``order_dict``. - """ - super().__init__(list(order_dict)) - self.order_dict = order_dict.copy() - self.default_order = default_order - - def reorder(self, iterable: Iterable[Package], - key: Callable[[Any], Package] | None = None) -> list[Package] | None: - package_name = self._get_package_name_from_iterable(iterable, key) - if package_name is None: - return None - - orderer = self.order_dict.get(package_name) - if orderer is None: - orderer = self.default_order - if orderer is None: - return None - - return orderer.reorder(iterable, key) - - def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: - orderer = self.order_dict.get(package_name) - if orderer is None: - if self.default_order is None: - # shouldn't get here, because applies_to should protect us... - raise RuntimeError( - "package family orderer %r does not apply to package family %r", - (self, package_name)) - - orderer = self.default_order - - return orderer.sort_key_implementation(package_name, version) - - def __str__(self) -> str: - items = sorted((x[0], str(x[1])) for x in self.order_dict.items()) - return str((items, str(self.default_order))) - - def __eq__(self, other): - return ( # noqa: E721 - type(other) == type(self) - and self.order_dict == other.order_dict - and self.default_order == other.default_order - ) - - def to_pod(self) -> dict[str, Any]: - """ - Example (in yaml): - - .. code-block:: yaml - - type: per_family - orderers: - - packages: ['foo', 'bah'] - type: version_split - first_version: '4.0.5' - - packages: ['python'] - type: sorted - descending: false - default_order: - type: sorted - descending: true - """ - orderers = {} - packages = {} - - # group package fams by orderer they use - for fam, orderer in self.order_dict.items(): - k = id(orderer) - orderers[k] = orderer - packages.setdefault(k, set()).add(fam) - - orderlist = [] - for k, fams in packages.items(): - orderer = orderers[k] - data = to_pod(orderer) - data["packages"] = sorted(fams) - orderlist.append(data) - - result: dict[str, Any] = {"orderers": orderlist} - - if self.default_order is not None: - result["default_order"] = to_pod(self.default_order) - - return result - - @classmethod - def from_pod(cls, data: dict[str, Any]) -> Self: - order_dict = {} - default_order = None - - for d in data["orderers"]: - d = d.copy() - fams = d.pop("packages") - orderer = from_pod(d) - - for fam in fams: - order_dict[fam] = orderer - - d = data.get("default_order") - if d: - default_order = from_pod(d) - - return cls(order_dict, default_order) +# Legacy orderer registry. Used as fallback by _find_orderer when an orderer +# is not found in the plugin system. +_orderers = {} -class VersionSplitPackageOrder(PackageOrder): - """Orders package versions <= a given version first. +def _find_orderer(name): + """Find an orderer class by name. - For example, given the versions [5, 4, 3, 2, 1], an orderer initialized - with ``version=3`` would give the order [3, 2, 1, 5, 4]. + Checks the plugin system first, then falls back to the _orderers + registry for orderers registered via register_orderer(). """ - name = "version_split" - - def __init__(self, first_version: Version, packages: list[str] | None = None) -> None: - """Create a reorderer. - - Args: - first_version (Version): Start with versions <= this value. - """ - super().__init__(packages) - self.first_version = first_version - - def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: - priority_key = 1 if version <= self.first_version else 0 - return priority_key, version - - def __str__(self) -> str: - return str(self.first_version) - - def __eq__(self, other): - return ( # noqa: E721 - type(other) == type(self) - and self.first_version == other.first_version - ) - - def to_pod(self) -> dict[str, Any]: - """ - Example (in yaml): - - .. code-block:: yaml - - type: version_split - first_version: "3.0.0" - packages: ["foo"] - """ - return dict( - first_version=str(self.first_version), - packages=self.packages, - ) - - @classmethod - def from_pod(cls, data: dict[str, Any]) -> Self: - return cls( - Version(data["first_version"]), - packages=data.get("packages"), - ) - - -class TimestampPackageOrder(PackageOrder): - """A timestamp order function. - - Given a time ``T``, this orderer returns packages released before ``T``, in descending - order, followed by those released after. If ``rank`` is non-zero, version - changes at that rank and above are allowed over the timestamp. - - For example, consider the common case where we want to prioritize packages - released before ``T``, except for newer patches. Consider the following package - versions, and time ``T``: - - .. code-block:: text - - 2.2.1 - 2.2.0 - 2.1.1 - 2.1.0 - 2.0.6 - 2.0.5 - <-- T - 2.0.0 - 1.9.0 - - A timestamp orderer set to ``rank=3`` (patch versions) will attempt to consume - the packages in the following order: - - .. code-block:: text - - 2.0.6 - 2.0.5 - 2.0.0 - 1.9.0 - 2.1.1 - 2.1.0 - 2.2.1 - 2.2.0 - - Notice that packages before ``T`` are preferred, followed by newer versions. - Newer versions are consumed in ascending order, except within rank (this is - why 2.1.1 is consumed before 2.1.0). + from rez.plugin_managers import plugin_manager + + try: + return plugin_manager.get_plugin_class("package_order", name) + except RezPluginError: + # Fallback to register_orderer() API-based registration + if name not in _orderers: + raise + return _orderers[name] + + +# Backward-compat aliases. These are resolved lazily via __getattr__ below to +# avoid a circular import: the plugin modules in rezplugins/package_order/ +# import from this module, so we cannot call _find_orderer (which loads the +# plugin system) at module load time. +# +# New code should import from the plugin system directly. +_LEGACY_ORDERER_NAMES = { + "NullPackageOrder": "no_order", + "PerFamilyOrder": "per_family", + "SortedOrder": "sorted", + "TimestampPackageOrder": "soft_timestamp", + "VersionSplitPackageOrder": "version_split", +} + + +def __getattr__(name): + """Provide backward-compat class aliases for orderers now in the plugin system. + + Resolved lazily to avoid circular imports at module load time. """ - name = "soft_timestamp" - - def __init__(self, timestamp: int, rank: int = 0, packages: list[str] | None = None) -> None: - """Create a reorderer. - - Args: - timestamp (int): Epoch time of timestamp. Packages before this time - are preferred. - rank (int): If non-zero, allow version changes at this rank or above - past the timestamp. - """ - super().__init__(packages) - self.timestamp = timestamp - self.rank = rank - - # dictionary mapping from package family to the first-version-after - # the given timestamp - self._cached_first_after = {} - self._cached_sort_key = {} - - def _get_first_after(self, package_family: str) -> Version | None: - """Get the first package version that is after the timestamp""" - try: - first_after = self._cached_first_after[package_family] - except KeyError: - first_after = self._calc_first_after(package_family) - self._cached_first_after[package_family] = first_after - return first_after - - def _calc_first_after(self, package_family: str) -> Version | None: - descending = sorted(iter_packages(package_family), - key=lambda p: p.version, - reverse=True) - first_after = None - for i, package in enumerate(descending): - if not package.timestamp: - continue - if package.timestamp > self.timestamp: - first_after = package.version - else: - break - - if not self.rank: - return first_after - - # if we have rank, then we need to then go back UP the - # versions, until we find one whose trimmed version doesn't - # match. - # Note that we COULD do this by simply iterating through - # an ascending sequence, in which case we wouldn't have to - # "switch direction" after finding the first result after - # by timestamp... but we're making the assumption that the - # timestamp break will be closer to the higher end of the - # version, and that we'll therefore have to check fewer - # timestamps this way... - trimmed_version = package.version.trim(self.rank - 1) - first_after = None - for after_package in reversed(descending[:i]): - if after_package.version.trim(self.rank - 1) != trimmed_version: - return after_package.version - - return first_after - - def _calc_sort_key(self, package_name: str, version: Version) -> SupportsLessThan: - first_after = self._get_first_after(package_name) - if first_after is None: - # all packages are before T - is_before: bool | int = True - else: - is_before = int(version < first_after) - - if is_before: - return is_before, version - - if self.rank: - return (is_before, - _ReversedComparable(version.trim(self.rank - 1)), - version.tokens[self.rank - 1:]) - - return is_before, _ReversedComparable(version) - - def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: - cache_key = (package_name, str(version)) - result = self._cached_sort_key.get(cache_key) - if result is None: - result = self._calc_sort_key(package_name, version) - self._cached_sort_key[cache_key] = result - - return result - - def __str__(self) -> str: - return str((self.timestamp, self.rank)) - - def __eq__(self, other): - return ( # noqa: E721 - type(other) == type(self) - and self.timestamp == other.timestamp - and self.rank == other.rank - ) - - def to_pod(self) -> dict[str, Any]: - """ - Example (in yaml): - - .. code-block:: yaml - - type: soft_timestamp - timestamp: 1234567 - rank: 3 - packages: ["foo"] - """ - return dict( - timestamp=self.timestamp, - rank=self.rank, - packages=self.packages, - ) - - @classmethod - def from_pod(cls, data: dict[str, Any]) -> Self: - return cls( - data["timestamp"], - rank=data.get("rank", 0), - packages=data.get("packages"), - ) + if name in _LEGACY_ORDERER_NAMES: + return _find_orderer(_LEGACY_ORDERER_NAMES[name]) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") class PackageOrderList(List[PackageOrder]): - """A list of package orderer. - """ + """A list of package orderer.""" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -657,6 +262,19 @@ def singleton(cls) -> PackageOrderList: """Filter list as configured by rezconfig.package_filter.""" return cls.from_pod(config.package_orderers) + @classmethod + def clear_singleton_cache(cls) -> None: + """Clear the cached singleton so the next access re-reads config. + + Use this when runtime configuration (e.g. REZ_PACKAGE_ORDERERS_JSON) + has changed and you want the new config to take effect. Note that + the config-level cache must also be cleared via + ``config._uncache("package_orderers")``. + """ + name = "_class_property_singleton" + if hasattr(cls, name): + delattr(cls, name) + @staticmethod def _to_orderer(orderer: dict | PackageOrder) -> PackageOrder: if isinstance(orderer, dict): @@ -727,12 +345,12 @@ def from_pod(data: dict[str, Any]) -> PackageOrder: data = data.copy() data.pop("type") - cls = _orderers[cls_name] + cls = _find_orderer(cls_name) return cls.from_pod(data) else: # old-style, kept for backwards compatibility cls_name, data_ = data - cls = _orderers[cls_name] + cls = _find_orderer(cls_name) return cls.from_pod(data_) @@ -744,28 +362,36 @@ def get_orderer(package_name: str, orderers: PackageOrderList | dict[str, Packag orderer = orderers.get(ALL_PACKAGES) if orderer is None: # default ordering is version descending - orderer = SortedOrder(descending=True) + sorted_order = _find_orderer("sorted") + orderer = sorted_order(descending=True) return orderer +# Orderers registered at runtime via register_orderer(). This is the API-based +# registration pathway, used as fallback by _find_orderer when an orderer is +# not found in the plugin system. +_orderers = {} + + +# Register an orderer for runtime/API-based use. Orderers registered here are +# found by _find_orderer as a fallback when the plugin system does not have +# a matching orderer. For filesystem-based, facility-wide orderers, prefer +# creating a plugin in rezplugins/package_order/ instead. def register_orderer(cls: type[PackageOrder]) -> bool: - """Register an orderer + """Register an orderer. + + This is the API-based registration pathway, useful for dynamic or + programmatic orderer registration. For persistent, facility-wide + orderers, prefer the plugin system (rezplugins/package_order/). Args: - cls (type[PackageOrder]): Package orderer class to register. + cls (type[PackageOrder]): Package order class to register. returns: bool: True if successfully registered, else False. """ - if isclass(cls) and issubclass(cls, PackageOrder) and \ - hasattr(cls, "name") and cls.name: + if isclass(cls) and issubclass(cls, PackageOrder) and hasattr(cls, "name") and cls.name: _orderers[cls.name] = cls return True else: return False - - -# registration of builtin orderers -_orderers = {} -for o in list(globals().values()): - register_orderer(o) diff --git a/src/rez/plugin_managers.py b/src/rez/plugin_managers.py index f0fc0e4107..d8e10c9434 100644 --- a/src/rez/plugin_managers.py +++ b/src/rez/plugin_managers.py @@ -5,6 +5,7 @@ """ Manages loading of all types of Rez plugins. """ + from __future__ import annotations from rez.config import config, expand_system_vars, _load_config_from_filepaths @@ -70,7 +71,7 @@ def extend_path(path, name): # frozen package. Return the path unchanged in that case. return path - pname = os.path.join(*name.split('.')) # Reconstitute as relative path + pname = os.path.join(*name.split(".")) # Reconstitute as relative path # Just in case os.extsep != '.' init_py = "__init__" + os.extsep + "py" path = path[:] @@ -106,13 +107,13 @@ class RezPluginType(object): 'type_name' must correspond with one of the source directories found under the 'plugins' directory. """ + type_name: str def __init__(self) -> None: if self.type_name is None: - raise TypeError("Subclasses of RezPluginType must provide a " - "'type_name' attribute") - self.pretty_type_name = self.type_name.replace('_', ' ') + raise TypeError("Subclasses of RezPluginType must provide a 'type_name' attribute") + self.pretty_type_name = self.type_name.replace("_", " ") self.plugin_classes: dict[str, type] = {} self.failed_plugins: dict[str, str] = {} self.plugin_modules: dict[str, types.ModuleType] = {} @@ -120,7 +121,7 @@ def __init__(self) -> None: self.load_plugins() def __repr__(self) -> str: - return '%s(%s)' % (self.__class__.__name__, self.plugin_classes.keys()) + return "%s(%s)" % (self.__class__.__name__, self.plugin_classes.keys()) def register_plugin(self, plugin_name: str, plugin_class: type, plugin_module: types.ModuleType) -> None: # TODO: check plugin_class to ensure it is a sub-class of expected base-class? @@ -136,7 +137,8 @@ def load_plugins(self) -> None: def load_plugins_from_namespace(self): import pkgutil from importlib import import_module - type_module_name = 'rezplugins.' + self.type_name + + type_module_name = "rezplugins." + self.type_name package = import_module(type_module_name) # on import, the `__path__` variable of the imported package is extended @@ -144,8 +146,7 @@ def load_plugins_from_namespace(self): # extend_path, above). this means that `walk_packages` will walk over all # modules on the search path at the same level (.e.g in a # 'rezplugins/type_name' sub-directory). - paths = [package.__path__] if isinstance(package.__path__, str) \ - else package.__path__ + paths = [package.__path__] if isinstance(package.__path__, str) else package.__path__ # reverse plugin path order, so that custom plugins have a chance to # be found before the builtin plugins (from /rezplugins). @@ -155,14 +156,12 @@ def load_plugins_from_namespace(self): if config.debug("plugins"): print_debug("searching plugin path %s...", path) - for importer, modname, ispkg in pkgutil.iter_modules( - [path], package.__name__ + '.'): - + for importer, modname, ispkg in pkgutil.iter_modules([path], package.__name__ + "."): if importer is None: continue - plugin_name = modname.split('.')[-1] - if plugin_name.startswith('_') or plugin_name == 'rezconfig': + plugin_name = modname.split(".")[-1] + if plugin_name.startswith("_") or plugin_name == "rezconfig": continue if plugin_name in self.plugin_modules: @@ -171,13 +170,11 @@ def load_plugins_from_namespace(self): # `sys.modules` below. skipping the rest of the process # for good. if config.debug("plugins"): - print_warning("skipped same named %s plugin at %s: %s" - % (self.type_name, path, modname)) + print_warning("skipped same named %s plugin at %s: %s" % (self.type_name, path, modname)) continue if config.debug("plugins"): - print_debug("loading %s plugin at %s: %s..." - % (self.type_name, path, modname)) + print_debug("loading %s plugin at %s: %s..." % (self.type_name, path, modname)) try: plugin_module = sys.modules.get(modname) @@ -202,8 +199,7 @@ def load_plugins_from_entry_points(self): for plugin in discovered_plugins: if config.debug("plugins"): - print_debug("loading %s plugin for %r..." - % (self.type_name, f"{plugin.name} = {plugin.value!r}")) + print_debug("loading %s plugin for %r..." % (self.type_name, f"{plugin.name} = {plugin.value!r}")) try: plugin_name = plugin.name plugin = plugin.load() @@ -214,7 +210,7 @@ def load_plugins_from_entry_points(self): self.print_log_plugins_error(plugin.__name__, e) def print_log_plugins_error(self, module_name, error): - nameish = module_name.split('.')[-1] + nameish = module_name.split(".")[-1] self.failed_plugins[nameish] = str(error) if not config.debug("plugins"): @@ -222,6 +218,7 @@ def print_log_plugins_error(self, module_name, error): import traceback from io import StringIO + out = StringIO() traceback.print_exc(file=out) print_debug(out.getvalue()) @@ -239,55 +236,46 @@ def register_plugin_module(self, plugin_name, plugin_module, plugin_path): print_warning( "plugin module %s is not loaded from current " "load path but reused from previous imported " - "path: %s" % (module_name, plugin_module.__file__)) - - if (hasattr(plugin_module, "register_plugin") - and callable(plugin_module.register_plugin)): + "path: %s" % (module_name, plugin_module.__file__) + ) + if hasattr(plugin_module, "register_plugin") and callable(plugin_module.register_plugin): plugin_class = plugin_module.register_plugin() if plugin_class is not None: - self.register_plugin( - plugin_name, - plugin_class, - plugin_module - ) + self.register_plugin(plugin_name, plugin_class, plugin_module) else: if config.debug("plugins"): print_warning( - "'register_plugin' function at %s: %s did " - "not return a class." % (plugin_path, module_name)) + "'register_plugin' function at %s: %s did not return a class." % (plugin_path, module_name) + ) else: if config.debug("plugins"): - print_warning( - "no 'register_plugin' function at %s: %s" - % (plugin_path, module_name)) + print_warning("no 'register_plugin' function at %s: %s" % (plugin_path, module_name)) def get_plugin_class(self, plugin_name: str) -> type: """Returns the class registered under the given plugin name.""" try: return self.plugin_classes[plugin_name] except KeyError: - raise RezPluginError("Unrecognised %s plugin: '%s'" - % (self.pretty_type_name, plugin_name)) + raise RezPluginError("Unrecognised %s plugin: '%s'" % (self.pretty_type_name, plugin_name)) def get_plugin_module(self, plugin_name: str) -> types.ModuleType: """Returns the module containing the plugin of the given name.""" try: return self.plugin_modules[plugin_name] except KeyError: - raise RezPluginError("Unrecognised %s plugin: '%s'" - % (self.pretty_type_name, plugin_name)) + raise RezPluginError("Unrecognised %s plugin: '%s'" % (self.pretty_type_name, plugin_name)) @cached_property def config_schema(self): """Returns the merged configuration data schema for this plugin type.""" from rez.config import _plugin_config_dict + d = _plugin_config_dict.get(self.type_name, {}) for name, plugin_class in self.plugin_classes.items(): - if hasattr(plugin_class, "schema_dict") \ - and plugin_class.schema_dict: + if hasattr(plugin_class, "schema_dict") and plugin_class.schema_dict: d_ = {name: plugin_class.schema_dict} deep_update(d, d_) return dict_to_schema(d, required=True, modifier=expand_system_vars) @@ -350,6 +338,7 @@ def register_plugin(): This is important because it ensures that rez's copy of 'rezplugins' is always found first. """ + def __init__(self) -> None: self._plugin_types: dict[str, LazySingleton[RezPluginType]] = {} @@ -390,15 +379,13 @@ def _get_plugin_type(self, plugin_type: str) -> RezPluginType: try: return self._plugin_types[plugin_type]() except KeyError: - raise RezPluginError("Unrecognised plugin type: '%s'" - % plugin_type) + raise RezPluginError("Unrecognised plugin type: '%s'" % plugin_type) def register_plugin_type(self, type_class: type[RezPluginType]) -> None: if not issubclass(type_class, RezPluginType): raise TypeError("'type_class' must be a RezPluginType sub class") if type_class.type_name is None: - raise TypeError("Subclasses of RezPluginType must provide a " - "'type_name' attribute") + raise TypeError("Subclasses of RezPluginType must provide a 'type_name' attribute") self._plugin_types[type_class.type_name] = LazySingleton(type_class) def get_plugin_types(self) -> list[str]: @@ -477,66 +464,72 @@ def create_instance(self, plugin_type: str, plugin_name, **instance_kwargs: Any) def get_summary_string(self) -> str: """Get a formatted string summarising the plugins that were loaded.""" - rows = [["PLUGIN TYPE", "NAME", "DESCRIPTION", "STATUS"], - ["-----------", "----", "-----------", "------"]] + rows = [["PLUGIN TYPE", "NAME", "DESCRIPTION", "STATUS"], ["-----------", "----", "-----------", "------"]] for plugin_type in sorted(self.get_plugin_types()): - type_name = plugin_type.replace('_', ' ') + type_name = plugin_type.replace("_", " ") for name in sorted(self.get_plugins(plugin_type)): module = self.get_plugin_module(plugin_type, name) - desc = (getattr(module, "__doc__", None) or '').strip() + desc = (getattr(module, "__doc__", None) or "").strip() rows.append((type_name, name, desc, "loaded")) - for (name, reason) in sorted(self.get_failed_plugins(plugin_type)): + for name, reason in sorted(self.get_failed_plugins(plugin_type)): msg = "FAILED: %s" % reason - rows.append((type_name, name, '', msg)) - return '\n'.join(columnise(rows)) + rows.append((type_name, name, "", msg)) + return "\n".join(columnise(rows)) # ------------------------------------------------------------------------------ # Plugin Types # ------------------------------------------------------------------------------ + class ShellPluginType(RezPluginType): - """Support for different types of target shells, such as bash, tcsh. - """ + """Support for different types of target shells, such as bash, tcsh.""" + type_name = "shell" class ReleaseVCSPluginType(RezPluginType): - """Support for different version control systems when releasing packages. - """ + """Support for different version control systems when releasing packages.""" + type_name = "release_vcs" class ReleaseHookPluginType(RezPluginType): - """Support for different version control systems when releasing packages. - """ + """Support for different version control systems when releasing packages.""" + type_name = "release_hook" class BuildSystemPluginType(RezPluginType): - """Support for different build systems when building packages. - """ + """Support for different build systems when building packages.""" + type_name = "build_system" class PackageRepositoryPluginType(RezPluginType): - """Support for different package repositories for loading packages. - """ + """Support for different package repositories for loading packages.""" + type_name = "package_repository" class BuildProcessPluginType(RezPluginType): - """Support for different build and release processes. - """ + """Support for different build and release processes.""" + type_name = "build_process" class CommandPluginType(RezPluginType): - """Support for different custom Rez applications/subcommands. - """ + """Support for different custom Rez applications/subcommands.""" + type_name = "command" +class PackageOrderPluginType(RezPluginType): + """Support for different ordering of packages.""" + + type_name = "package_order" + + plugin_manager = RezPluginManager() @@ -547,3 +540,4 @@ class CommandPluginType(RezPluginType): plugin_manager.register_plugin_type(PackageRepositoryPluginType) plugin_manager.register_plugin_type(BuildProcessPluginType) plugin_manager.register_plugin_type(CommandPluginType) +plugin_manager.register_plugin_type(PackageOrderPluginType) diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py index ad984d8b32..dfc119c858 100644 --- a/src/rez/tests/test_completion.py +++ b/src/rez/tests/test_completion.py @@ -5,6 +5,7 @@ """ test completions """ + import unittest from rez.tests.util import TestBase from rez.config import Config, get_module_root_config @@ -15,52 +16,74 @@ class TestCompletion(TestBase): @classmethod def setUpClass(cls) -> None: packages_path = cls.data_path("solver", "packages") - cls.settings = dict( - packages_path=[packages_path], - package_filter=None) + cls.settings = dict(packages_path=[packages_path], package_filter=None) cls.config = Config([get_module_root_config()], locked=True) def test_config(self) -> None: """Test config completion.""" + def _eq(prefix, expected_completions) -> None: completions = self.config.get_completions(prefix) self.assertEqual(set(completions), set(expected_completions)) _eq("zzz", []) _eq("pref", ["prefix_prompt"]) - _eq("plugin", ["plugins", - "plugin_path"]) - _eq("plugins", ["plugins", - "plugins.command", - "plugins.package_repository", - "plugins.build_process", - "plugins.build_system", - "plugins.release_hook", - "plugins.release_vcs", - "plugins.shell"]) - _eq("plugins.release_vcs.releasable_", - ["plugins.release_vcs.releasable_branches"]) + _eq("plugin", ["plugins", "plugin_path"]) + _eq( + "plugins", + [ + "plugins", + "plugins.command", + "plugins.package_order", + "plugins.package_repository", + "plugins.build_process", + "plugins.build_system", + "plugins.release_hook", + "plugins.release_vcs", + "plugins.shell", + ], + ) + _eq("plugins.release_vcs.releasable_", ["plugins.release_vcs.releasable_branches"]) def test_packages(self) -> None: """Test packages completion.""" + def _eq(prefix, expected_completions) -> None: completions = get_completions(prefix) self.assertEqual(set(completions), set(expected_completions)) _eq("zzz", []) - _eq("", ["bahish", "nada", "nopy", "pybah", "pydad", "pyfoo", "pymum", - "pyodd", "pyson", "pysplit", "python", "pyvariants", - "test_variant_split_start", "test_variant_split_mid1", - "test_variant_split_mid2", "test_variant_split_end", "missing_variant_requires", - "test_weakly_reference_requires", "test_weakly_reference_variant"]) - _eq("py", ["pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", - "pysplit", "python", "pyvariants"]) + _eq( + "", + [ + "bahish", + "nada", + "nopy", + "pybah", + "pydad", + "pyfoo", + "pymum", + "pyodd", + "pyson", + "pysplit", + "python", + "pyvariants", + "test_variant_split_start", + "test_variant_split_mid1", + "test_variant_split_mid2", + "test_variant_split_end", + "missing_variant_requires", + "test_weakly_reference_requires", + "test_weakly_reference_variant", + ], + ) + _eq("py", ["pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", "pysplit", "python", "pyvariants"]) _eq("pys", ["pyson", "pysplit"]) _eq("pyb", ["pybah", "pybah-4", "pybah-5"]) _eq("pybah-", ["pybah-4", "pybah-5"]) _eq("pyfoo-3.0", ["pyfoo-3.0.0"]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/src/rez/tests/test_packages_order.py b/src/rez/tests/test_packages_order.py index 12c0ebe7c3..fb7b9bc3fe 100644 --- a/src/rez/tests/test_packages_order.py +++ b/src/rez/tests/test_packages_order.py @@ -5,18 +5,54 @@ """ Test cases for package_order.py (package ordering) """ + import json from rez.config import config -from rez.package_order import NullPackageOrder, PackageOrder, PerFamilyOrder, VersionSplitPackageOrder, \ - TimestampPackageOrder, SortedOrder, PackageOrderList, from_pod +from rez.package_order import ( + PackageOrder, + PackageOrderList, + from_pod, + to_pod, + get_orderer, + register_orderer, + _find_orderer, + _orderers, + FallbackComparable, +) from rez.packages import iter_packages from rez.tests.util import TestBase, TempdirMixin -from rez.version import Version +from rez.version import Version, VersionRange + + +def _orderer(name): + """Resolve an orderer class lazily. + + The plugin manager may be reset by other tests (e.g. test_plugin_manager), + which re-imports plugin modules and creates new class objects. Importing + orderer classes at module level binds them to stale class objects. This + helper resolves them at call time so tests always see the current class. + """ + from rez.package_order import _find_orderer + + return _find_orderer(name) class _BaseTestPackagesOrder(TestBase, TempdirMixin): """Base class for a package ordering test case""" + + def setUp(self) -> None: + # Re-resolve orderer classes on each test run. The plugin manager + # may be reset by other tests (e.g. test_plugin_manager), which + # re-imports plugin modules and creates new class objects. Class + # references bound at module import time would become stale. + self.NullPackageOrder = _orderer("no_order") + self.SortedOrder = _orderer("sorted") + self.PerFamilyOrder = _orderer("per_family") + self.VersionSplitPackageOrder = _orderer("version_split") + self.TimestampPackageOrder = _orderer("soft_timestamp") + super().setUp() + @classmethod def setUpClass(cls) -> None: TempdirMixin.setUpClass() @@ -24,12 +60,7 @@ def setUpClass(cls) -> None: cls.py_packages_path = cls.data_path("packages", "py_packages") cls.solver_packages_path = cls.data_path("solver", "packages") - cls.settings = dict( - packages_path=[ - cls.solver_packages_path, - cls.py_packages_path - ], - package_filter=None) + cls.settings = dict(packages_path=[cls.solver_packages_path, cls.py_packages_path], package_filter=None) @classmethod def tearDownClass(cls) -> None: @@ -73,27 +104,24 @@ class TestNullPackageOrder(_BaseTestPackagesOrder): def test_repr(self) -> None: """Validate we can represent a VersionSplitPackageOrder as a string.""" - self.assertEqual("NullPackageOrder({})", repr(NullPackageOrder())) + self.assertEqual("NullPackageOrder({})", repr(self.NullPackageOrder())) def test_comparison(self) -> None: """Validate we can compare VersionSplitPackageOrder together.""" - inst1 = NullPackageOrder() - inst2 = NullPackageOrder() + inst1 = self.NullPackageOrder() + inst2 = self.NullPackageOrder() self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == "wrong_type") # __eq__ negative (wrong type) self.assertTrue(inst1 != "wrong_type") # __ne__ positive (wrong type) self.assertFalse(inst1 != inst2) # __ne__ negative def test_pod(self) -> None: - """Validate we can save and load a VersionSplitPackageOrder to it's pod representation.""" - self._test_pod(NullPackageOrder()) + """Validate we can save and load a VersionSplitPackageOrder to its pod representation.""" + self._test_pod(self.NullPackageOrder()) def test_sha1(self) -> None: - """Validate we can get a sha1 hash. - """ - self.assertEqual( - 'bf7c2fa4e6bd198c02adeea2c3a382cf57242051', NullPackageOrder().sha1 - ) + """Validate we can get a sha1 hash.""" + self.assertEqual("bf7c2fa4e6bd198c02adeea2c3a382cf57242051", self.NullPackageOrder().sha1) class TestSortedOrder(_BaseTestPackagesOrder): @@ -101,17 +129,17 @@ class TestSortedOrder(_BaseTestPackagesOrder): def test_reorder_ascending(self) -> None: """Validate we can sort packages in ascending order.""" - self._test_reorder(SortedOrder(descending=False), "pymum", ["1", "2", "3"]) + self._test_reorder(self.SortedOrder(descending=False), "pymum", ["1", "2", "3"]) def test_reorder_descending(self) -> None: """Validate we can sort packages in descending order.""" - self._test_reorder(SortedOrder(descending=True), "pymum", ["3", "2", "1"]) + self._test_reorder(self.SortedOrder(descending=True), "pymum", ["3", "2", "1"]) def test_comparison(self) -> None: """Validate we can compare SortedOrder together.""" - inst1 = SortedOrder(descending=False) - inst2 = SortedOrder(descending=False) - inst3 = SortedOrder(descending=True) + inst1 = self.SortedOrder(descending=False) + inst2 = self.SortedOrder(descending=False) + inst3 = self.SortedOrder(descending=True) self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == inst3) # __eq__ negative self.assertTrue(inst1 != inst3) # __ne__ positive @@ -121,11 +149,11 @@ def test_comparison(self) -> None: def test_repr(self) -> None: """Validate we can represent a SortedOrder as a string.""" - self.assertEqual("SortedOrder(True)", repr(SortedOrder(descending=True))) + self.assertEqual("SortedOrder(True)", repr(self.SortedOrder(descending=True))) def test_pod(self) -> None: - """Validate we can save and load a SortedOrder to it's pod representation.""" - self._test_pod(SortedOrder(descending=True)) + """Validate we can save and load a SortedOrder to its pod representation.""" + self._test_pod(self.SortedOrder(descending=True)) class TestPerFamilyOrder(_BaseTestPackagesOrder): @@ -137,13 +165,13 @@ def test_reorder(self) -> None: expected_split_result = ["2.6.0", "2.5.2", "2.7.0", "2.6.8"] expected_timestamp_result = ["1.1.1", "1.1.0", "1.0.6", "1.0.5", "1.2.0", "2.0.0", "2.1.5", "2.1.0"] - orderer = PerFamilyOrder( + orderer = self.PerFamilyOrder( order_dict=dict( - pysplit=NullPackageOrder(), - python=VersionSplitPackageOrder(Version("2.6.0")), - timestamped=TimestampPackageOrder(timestamp=3001, rank=3) + pysplit=self.NullPackageOrder(), + python=self.VersionSplitPackageOrder(Version("2.6.0")), + timestamped=self.TimestampPackageOrder(timestamp=3001, rank=3), ), - default_order=SortedOrder(descending=False) + default_order=self.SortedOrder(descending=False), ) self._test_reorder(orderer, "pysplit", expected_null_result) @@ -153,20 +181,36 @@ def test_reorder(self) -> None: def test_reorder_no_packages(self) -> None: """Validate ordering for a family with no packages.""" - orderer = PerFamilyOrder(order_dict=dict(missing_package=NullPackageOrder())) + orderer = self.PerFamilyOrder(order_dict=dict(missing_package=self.NullPackageOrder())) self._test_reorder(orderer, "missing_package", []) def test_reorder_no_default_order(self) -> None: - """Test behavior when there's no secondary default_order.""" - fam_orderer = PerFamilyOrder(order_dict={}) + """Validate we correctly handle no default order for a PerFamilyOrder.""" + fam_orderer = self.PerFamilyOrder(order_dict={}) self._test_reorder(fam_orderer, "pymum", ["3", "2", "1"]) + def test_sort_key_implementation_with_default_order(self) -> None: + """sort_key_implementation falls back to default_order for unregistered families.""" + orderer = self.PerFamilyOrder( + order_dict={}, + default_order=self.SortedOrder(descending=False), + ) + # "unknown" is not in order_dict, so falls back to default_order + key = orderer.sort_key_implementation("unknown", Version("1.0")) + self.assertIsNotNone(key) + + def test_sort_key_implementation_no_orderer_raises(self) -> None: + """sort_key_implementation raises RuntimeError when no orderer applies.""" + orderer = self.PerFamilyOrder(order_dict={}, default_order=None) + with self.assertRaises(RuntimeError): + orderer.sort_key_implementation("unknown", Version("1.0")) + def test_comparison(self) -> None: """Validate we can compare PerFamilyOrder.""" - inst1 = PerFamilyOrder(order_dict={'foo': NullPackageOrder()}, default_order=NullPackageOrder()) - inst2 = PerFamilyOrder(order_dict={'foo': NullPackageOrder()}, default_order=NullPackageOrder()) - inst3 = PerFamilyOrder(order_dict={'bar': NullPackageOrder()}, default_order=NullPackageOrder()) - inst4 = PerFamilyOrder(order_dict={'foo': NullPackageOrder()}, default_order=None) + inst1 = self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()}, default_order=self.NullPackageOrder()) + inst2 = self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()}, default_order=self.NullPackageOrder()) + inst3 = self.PerFamilyOrder(order_dict={"bar": self.NullPackageOrder()}, default_order=self.NullPackageOrder()) + inst4 = self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()}, default_order=None) self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == inst3) # __eq__ negative (different order dict) self.assertFalse(inst1 == inst4) # __eq__ negative (different default_order) @@ -176,19 +220,17 @@ def test_comparison(self) -> None: def test_repr(self) -> None: """Validate we can represent a PerFamilyOrder as a string.""" - inst = PerFamilyOrder(order_dict={"family1": VersionSplitPackageOrder(Version("2.6.0"))}) + inst = self.PerFamilyOrder(order_dict={"family1": self.VersionSplitPackageOrder(Version("2.6.0"))}) self.assertEqual("PerFamilyOrder(([('family1', '2.6.0')], 'None'))", repr(inst)) def test_pod(self) -> None: - """Validate we can save and load a PerFamilyOrder to it's pod representation.""" + """Validate we can save and load a PerFamilyOrder to its pod representation.""" self._test_pod( - PerFamilyOrder(order_dict={'foo': NullPackageOrder()}, default_order=NullPackageOrder()) + self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()}, default_order=self.NullPackageOrder()) ) # No default_order - self._test_pod( - PerFamilyOrder(order_dict={'foo': NullPackageOrder()}) - ) + self._test_pod(self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()})) class TestVersionSplitPackageOrder(_BaseTestPackagesOrder): @@ -196,15 +238,15 @@ class TestVersionSplitPackageOrder(_BaseTestPackagesOrder): def test_reordere(self) -> None: """Validate package ordering with a VersionSplitPackageOrder""" - orderer = VersionSplitPackageOrder(Version("2.6.0")) + orderer = self.VersionSplitPackageOrder(Version("2.6.0")) expected = ["2.6.0", "2.5.2", "2.7.0", "2.6.8"] self._test_reorder(orderer, "python", expected) def test_comparison(self) -> None: """Validate we can compare VersionSplitPackageOrder together.""" - inst1 = VersionSplitPackageOrder(first_version=Version("1.2.3")) - inst2 = VersionSplitPackageOrder(first_version=Version("1.2.3")) - inst3 = VersionSplitPackageOrder(first_version=Version("1.2.4")) + inst1 = self.VersionSplitPackageOrder(first_version=Version("1.2.3")) + inst2 = self.VersionSplitPackageOrder(first_version=Version("1.2.3")) + inst3 = self.VersionSplitPackageOrder(first_version=Version("1.2.4")) self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == inst3) # __eq__ negative self.assertTrue(inst1 != inst3) # __ne__ positive @@ -214,12 +256,12 @@ def test_comparison(self) -> None: def test_repr(self) -> None: """Validate we can represent a VersionSplitPackageOrder as a string.""" - inst = VersionSplitPackageOrder(first_version=Version("1,2,3")) + inst = self.VersionSplitPackageOrder(first_version=Version("1,2,3")) self.assertEqual("VersionSplitPackageOrder(1,2,3)", repr(inst)) def test_pod(self) -> None: - """Validate we can save and load a VersionSplitPackageOrder to it's pod representation.""" - self._test_pod(VersionSplitPackageOrder(first_version=Version("1.2.3"))) + """Validate we can save and load a VersionSplitPackageOrder to its pod representation.""" + self._test_pod(self.VersionSplitPackageOrder(first_version=Version("1.2.3"))) class TestTimestampPackageOrder(_BaseTestPackagesOrder): @@ -227,51 +269,51 @@ class TestTimestampPackageOrder(_BaseTestPackagesOrder): def test_reorder_no_rank(self) -> None: """Validate reordering with a rank of 0.""" - orderer = TimestampPackageOrder(timestamp=3001) - expected = ['1.1.0', '1.0.6', '1.0.5', '1.1.1', '1.2.0', '2.0.0', '2.1.0', '2.1.5'] + orderer = self.TimestampPackageOrder(timestamp=3001) + expected = ["1.1.0", "1.0.6", "1.0.5", "1.1.1", "1.2.0", "2.0.0", "2.1.0", "2.1.5"] self._test_reorder(orderer, "timestamped", expected) def test_reorder_rank_3(self) -> None: """Validate reordering with a rank of 3.""" # after v1.1.0 and before v1.1.1 - orderer1 = TimestampPackageOrder(timestamp=3001, rank=3) + orderer1 = self.TimestampPackageOrder(timestamp=3001, rank=3) expected1 = ["1.1.1", "1.1.0", "1.0.6", "1.0.5", "1.2.0", "2.0.0", "2.1.5", "2.1.0"] self._test_reorder(orderer1, "timestamped", expected1) # after v2.1.0 and before v2.1.5 - orderer2 = TimestampPackageOrder(timestamp=7001, rank=3) + orderer2 = self.TimestampPackageOrder(timestamp=7001, rank=3) expected2 = ["2.1.5", "2.1.0", "2.0.0", "1.2.0", "1.1.1", "1.1.0", "1.0.6", "1.0.5"] self._test_reorder(orderer2, "timestamped", expected2) def test_reorder_rank_2(self) -> None: """Add coverage for a corner case where there's only one candidate without the rank.""" - orderer = TimestampPackageOrder(timestamp=4001, rank=3) # 1.1.1 - expected = ['1.1.1', '1.1.0', '1.0.6', '1.0.5', '1.2.0', '2.0.0', '2.1.5', '2.1.0'] + orderer = self.TimestampPackageOrder(timestamp=4001, rank=3) # 1.1.1 + expected = ["1.1.1", "1.1.0", "1.0.6", "1.0.5", "1.2.0", "2.0.0", "2.1.5", "2.1.0"] self._test_reorder(orderer, "timestamped", expected) def test_reorder_packages_without_timestamps(self) -> None: """Validate reordering of packages that have no timestamp data.""" - orderer = TimestampPackageOrder(timestamp=3001) + orderer = self.TimestampPackageOrder(timestamp=3001) self._test_reorder(orderer, "pymum", ["3", "2", "1"]) def test_reorder_all_packages_before_timestamp(self) -> None: """Test behavior when all packages are before the timestamp.""" - timestamp_orderer = TimestampPackageOrder(timestamp=9999999999, rank=3) - expected = ['2.1.5', '2.1.0', '2.0.0', '1.2.0', '1.1.1', '1.1.0', '1.0.6', '1.0.5'] + timestamp_orderer = self.TimestampPackageOrder(timestamp=9999999999, rank=3) + expected = ["2.1.5", "2.1.0", "2.0.0", "1.2.0", "1.1.1", "1.1.0", "1.0.6", "1.0.5"] self._test_reorder(timestamp_orderer, "timestamped", expected) def test_reorder_all_packages_after_timestamp(self) -> None: """Test behavior when all packages are after the timestamp.""" - timestamp_orderer = TimestampPackageOrder(timestamp=0, rank=3) - expected = ['1.0.6', '1.0.5', '1.1.1', '1.1.0', '1.2.0', '2.0.0', '2.1.5', '2.1.0'] + timestamp_orderer = self.TimestampPackageOrder(timestamp=0, rank=3) + expected = ["1.0.6", "1.0.5", "1.1.1", "1.1.0", "1.2.0", "2.0.0", "2.1.5", "2.1.0"] self._test_reorder(timestamp_orderer, "timestamped", expected) def test_comparison(self) -> None: """Validate we can compare TimestampPackageOrder.""" - inst1 = TimestampPackageOrder(timestamp=1, rank=1) - inst2 = TimestampPackageOrder(timestamp=1, rank=1) - inst3 = TimestampPackageOrder(timestamp=2, rank=1) - inst4 = TimestampPackageOrder(timestamp=2, rank=2) + inst1 = self.TimestampPackageOrder(timestamp=1, rank=1) + inst2 = self.TimestampPackageOrder(timestamp=1, rank=1) + inst3 = self.TimestampPackageOrder(timestamp=2, rank=1) + inst4 = self.TimestampPackageOrder(timestamp=2, rank=2) self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == inst3) # __eq__ negative (different timestamp) self.assertFalse(inst1 == inst4) # __eq__ negative (different rank) @@ -281,12 +323,12 @@ def test_comparison(self) -> None: def test_repr(self) -> None: """Validate we can represent a TimestampPackageOrder as a string.""" - inst = TimestampPackageOrder(timestamp=1, rank=2) + inst = self.TimestampPackageOrder(timestamp=1, rank=2) self.assertEqual(repr(inst), "TimestampPackageOrder((1, 2))") def test_pod(self) -> None: """Validate we can save and load a TimestampPackageOrder to pod representation.""" - self._test_pod(TimestampPackageOrder(timestamp=3001, rank=3)) + self._test_pod(self.TimestampPackageOrder(timestamp=3001, rank=3)) class TestPackageOrdererList(_BaseTestPackagesOrder): @@ -294,26 +336,21 @@ class TestPackageOrdererList(_BaseTestPackagesOrder): def test_singleton(self) -> None: """Validate we can build a PackageOrderList object from configuration values.""" - config.override("package_orderers", [ - { - "type": "per_family", - "orderers": [ - { - "packages": ["python"], - "type": "version_split", - "first_version": "2.9.9" - } - ] - } - ]) + config.override( + "package_orderers", + [ + { + "type": "per_family", + "orderers": [{"packages": ["python"], "type": "version_split", "first_version": "2.9.9"}], + } + ], + ) expected = PackageOrderList() - expected.append(PerFamilyOrder(order_dict={ - "python": VersionSplitPackageOrder(Version("2.9.9")) - })) + expected.append(self.PerFamilyOrder(order_dict={"python": self.VersionSplitPackageOrder(Version("2.9.9"))})) # Clear @classproperty cache try: - delattr(PackageOrderList, '_class_property_singleton') + delattr(PackageOrderList, "_class_property_singleton") except AttributeError: pass self.assertEqual(expected, PackageOrderList.singleton) @@ -324,7 +361,7 @@ def test_singleton_novalue(self) -> None: # Clear @classproperty cache try: - delattr(PackageOrderList, '_class_property_singleton') + delattr(PackageOrderList, "_class_property_singleton") except AttributeError: pass @@ -332,19 +369,424 @@ def test_singleton_novalue(self) -> None: def test_pod(self) -> None: """Validate we can save and load a PackageOrdererList to pod representation.""" - inst = PackageOrderList(( - VersionSplitPackageOrder(Version("2.6.0")), - PerFamilyOrder(order_dict={}, default_order=SortedOrder(descending=False)) - )) + inst = PackageOrderList( + ( + self.VersionSplitPackageOrder(Version("2.6.0")), + self.PerFamilyOrder(order_dict={}, default_order=self.SortedOrder(descending=False)), + ) + ) self._test_pod(inst) + def test_from_pod_module_function_round_trip(self): + """Verify the module-level from_pod function resolves orderer types.""" + from rez.package_order import from_pod as module_from_pod + + # sorted + orderer = module_from_pod({"type": "sorted", "descending": True}) + self.assertIsInstance(orderer, self.SortedOrder) + self.assertTrue(orderer.descending) + + # no_order + orderer = module_from_pod({"type": "no_order"}) + self.assertIsInstance(orderer, self.NullPackageOrder) + + # version_split + orderer = module_from_pod({"type": "version_split", "first_version": "1.2.3"}) + self.assertIsInstance(orderer, self.VersionSplitPackageOrder) + + def test_isinstance_against_imported_classes(self): + """Verify isinstance works for orderer classes imported from package_order.""" + self.assertIsInstance(self.SortedOrder(descending=True), self.SortedOrder) + self.assertIsInstance(self.NullPackageOrder(), self.NullPackageOrder) + self.assertIsInstance( + self.VersionSplitPackageOrder(first_version=Version("1.0")), self.VersionSplitPackageOrder + ) + self.assertIsInstance(self.TimestampPackageOrder(timestamp=1000), self.TimestampPackageOrder) + # Cross-type checks + self.assertNotIsInstance(self.SortedOrder(descending=True), self.NullPackageOrder) + + def test_get_orderer_default_fallback(self): + """Verify get_orderer falls back to SortedOrder(descending=True).""" + from rez.package_order import get_orderer + + config.override("package_orderers", None) + # Clear singleton cache + try: + delattr(PackageOrderList, "_class_property_singleton") + except AttributeError: + pass + + orderer = get_orderer("nonexistent_package") + self.assertIsInstance(orderer, self.SortedOrder) + self.assertTrue(orderer.descending) + + def test_plugin_system_loads_builtin_orderers(self) -> None: + """Verify all five built-in orderers are loadable via the plugin system.""" + from rez.plugin_managers import plugin_manager + + expected = { + "no_order": "NullPackageOrder", + "sorted": "SortedOrder", + "per_family": "PerFamilyOrder", + "version_split": "VersionSplitPackageOrder", + "soft_timestamp": "TimestampPackageOrder", + } + + for plugin_name, class_name in expected.items(): + cls = plugin_manager.get_plugin_class("package_order", plugin_name) + self.assertEqual(cls.__name__, class_name) + + def test_pod_round_trip_through_plugin_system(self) -> None: + """Verify to_pod/from_pod round-trip works through the plugin system.""" + from rez.package_order import to_pod, from_pod + + orderers = [ + self.SortedOrder(descending=True), + self.NullPackageOrder(), + self.VersionSplitPackageOrder(first_version=Version("1.2.3")), + self.TimestampPackageOrder(timestamp=3001, rank=3), + self.PerFamilyOrder(order_dict={"foo": self.NullPackageOrder()}, default_order=self.NullPackageOrder()), + ] + + for original in orderers: + pod = to_pod(original) + restored = from_pod(pod) + self.assertEqual(original, restored) + self.assertIs(type(original), type(restored)) + + def test_legacy_register_orderer_fallback(self) -> None: + """Verify _find_orderer falls back to _orderers for legacy-registered orderers.""" + from rez.package_order import PackageOrder, register_orderer, _find_orderer, _orderers + + class TestLegacyOrderer(PackageOrder): + name = "test_legacy_fallback" + + def sort_key_implementation(self, package_name, version): + return 0 + + def __str__(self): + return "test" + + def to_pod(self): + return {} + + @classmethod + def from_pod(cls, data): + return cls() + + try: + register_orderer(TestLegacyOrderer) + found = _find_orderer("test_legacy_fallback") + self.assertIs(found, TestLegacyOrderer) + finally: + _orderers.pop("test_legacy_fallback", None) + + def test_rez_package_orderers_json_env_var(self) -> None: + """Verify REZ_PACKAGE_ORDERERS_JSON configures orderers at runtime. + + The test framework uses a locked config that blocks env-var reads, so + we simulate the env-var path by parsing the JSON and using + config.override() — this is exactly what the _JSON path does. + """ + import json + + config_json = json.dumps( + [ + { + "type": "per_family", + "orderers": [{"packages": ["python"], "type": "version_split", "first_version": "2.9.9"}], + } + ] + ) + + old_overrides = config.overrides.get("package_orderers") + try: + config.override("package_orderers", json.loads(config_json)) + PackageOrderList.clear_singleton_cache() + + orderers = PackageOrderList.singleton + self.assertEqual(len(orderers), 1) + self.assertEqual(orderers[0].name, "per_family") + finally: + if old_overrides is not None: + config.override("package_orderers", old_overrides) + else: + config.override("package_orderers", None) + PackageOrderList.clear_singleton_cache() + + def test_cache_invalidation_for_env_var_changes(self) -> None: + """Verify cache invalidation allows config changes to take effect.""" + import json + + old_overrides = config.overrides.get("package_orderers") + try: + # First config: empty orderers + config.override("package_orderers", None) + PackageOrderList.clear_singleton_cache() + orderers1 = PackageOrderList.singleton + self.assertEqual(len(orderers1), 0) + + # Second config: one orderer + config.override("package_orderers", json.loads('[{"type": "sorted", "descending": false}]')) + PackageOrderList.clear_singleton_cache() + orderers2 = PackageOrderList.singleton + self.assertEqual(len(orderers2), 1) + self.assertEqual(orderers2[0].name, "sorted") + self.assertFalse(orderers2[0].descending) + finally: + if old_overrides is not None: + config.override("package_orderers", old_overrides) + else: + config.override("package_orderers", None) + PackageOrderList.clear_singleton_cache() + + def test_kwarg_orderer_configurable_via_env_var(self) -> None: + """Verify an orderer with constructor kwargs can be configured at runtime. + + This validates the pattern used by PRs #1706 (PEP440PackageOrder with + 'prerelease' kwarg) and #1709 (CustomPackageOrder with 'packages' kwarg) + without requiring those PRs to be merged. + + The test framework uses a locked config that blocks env-var reads, so + we simulate the _JSON path by parsing JSON and using config.override(). + """ + import json + from rez.package_order import PackageOrder, register_orderer, _orderers + + class TestKwargOrderer(PackageOrder): + name = "test_kwarg" + + def __init__(self, mode="default", packages=None): + super().__init__(packages) + self.mode = mode + + def sort_key_implementation(self, package_name, version): + return 0 + + def __str__(self): + return str(self.mode) + + def __eq__(self, other): + return type(self) is type(other) and self.mode == other.mode + + def to_pod(self): + return {"mode": self.mode, "packages": self.packages} + + @classmethod + def from_pod(cls, data): + return cls( + mode=data.get("mode", "default"), + packages=data.get("packages"), + ) + + old_overrides = config.overrides.get("package_orderers") + try: + register_orderer(TestKwargOrderer) + + config.override( + "package_orderers", json.loads('[{"type": "test_kwarg", "mode": "production", "packages": ["foo"]}]') + ) + PackageOrderList.clear_singleton_cache() + + orderers = PackageOrderList.singleton + self.assertEqual(len(orderers), 1) + self.assertEqual(orderers[0].name, "test_kwarg") + self.assertEqual(orderers[0].mode, "production") + self.assertEqual(orderers[0].packages, ["foo"]) + finally: + _orderers.pop("test_kwarg", None) + if old_overrides is not None: + config.override("package_orderers", old_overrides) + else: + config.override("package_orderers", None) + PackageOrderList.clear_singleton_cache() + class TestPackageOrderPublic(TestBase): """Additional tests for public symbols in package_order.py""" + def setUp(self) -> None: + self.VersionSplitPackageOrder = _orderer("version_split") + self.SortedOrder = _orderer("sorted") + self.NullPackageOrder = _orderer("no_order") + self.PerFamilyOrder = _orderer("per_family") + super().setUp() + def test_from_pod_old_style(self) -> None: """Validate from_pod is still compatible with the older pod style.""" self.assertEqual( - VersionSplitPackageOrder(first_version=Version("1.2.3")), - from_pod(("version_split", {"first_version": "1.2.3"})) + self.VersionSplitPackageOrder(first_version=Version("1.2.3")), + from_pod(("version_split", {"first_version": "1.2.3"})), ) + + def test_find_orderer_unknown_name(self) -> None: + """_find_orderer raises RezPluginError for unknown orderer names.""" + from rez.exceptions import RezPluginError + + with self.assertRaises(RezPluginError): + _find_orderer("nonexistent_orderer") + + def test_getattr_unknown_name(self) -> None: + """Module-level __getattr__ raises AttributeError for unknown names.""" + import rez.package_order as po + + with self.assertRaises(AttributeError): + po.nonexistent_attribute # noqa: B018 + + def test_register_orderer_invalid(self) -> None: + """register_orderer returns False for non-PackageOrder classes.""" + + class NotAnOrderer: + pass + + self.assertFalse(register_orderer(NotAnOrderer)) + + def test_register_orderer_valid(self) -> None: + """register_orderer returns True and registers a valid orderer.""" + + class TestOrderer(PackageOrder): + name = "test_coverage_orderer" + + def sort_key_implementation(self, package_name, version): + return 0 + + def __str__(self): + return "test" + + def to_pod(self): + return {} + + @classmethod + def from_pod(cls, data): + return cls() + + try: + self.assertTrue(register_orderer(TestOrderer)) + self.assertEqual(_find_orderer("test_coverage_orderer"), TestOrderer) + finally: + _orderers.pop("test_coverage_orderer", None) + + def test_sort_key_with_version_range(self) -> None: + """sort_key handles VersionRange, not just Version.""" + orderer = self.SortedOrder(descending=True) + key = orderer.sort_key("foo", VersionRange("1.0+<2.0")) + self.assertIsInstance(key, tuple) + + def test_sort_key_with_none(self) -> None: + """sort_key returns 0 for None (preserves original order).""" + orderer = self.SortedOrder(descending=True) + self.assertEqual(orderer.sort_key("foo", None), 0) + + def test_sort_key_invalid_type(self) -> None: + """sort_key raises TypeError for unrecognized types.""" + orderer = self.SortedOrder(descending=True) + with self.assertRaises(TypeError): + orderer.sort_key("foo", 123) + + def test_fallback_comparable(self) -> None: + """FallbackComparable falls back when main comparison fails.""" + + class _Raises: + """Object that raises on comparison, forcing fallback path.""" + + def __eq__(self, other): + raise TypeError("nope") + + def __lt__(self, other): + raise TypeError("nope") + + fc1 = FallbackComparable(_Raises(), 1) + fc2 = FallbackComparable(_Raises(), 1) + # main comparison raises, so falls back to 1 == 1 + self.assertTrue(fc1 == fc2) + + fc3 = FallbackComparable(_Raises(), 2) + self.assertFalse(fc1 == fc3) + self.assertTrue(fc1 < fc3) + + def test_fallback_comparable_repr(self) -> None: + """FallbackComparable has a useful repr.""" + fc = FallbackComparable(1, 2) + self.assertIn("FallbackComparable", repr(fc)) + + def test_get_orderer_default(self) -> None: + """get_orderer returns descending SortedOrder as default fallback.""" + orderer = get_orderer("nonexistent_package") + self.assertTrue(orderer.descending) + + def test_to_pod_round_trip(self) -> None: + """to_pod produces a dict with 'type' that from_pod can consume.""" + orderer = self.SortedOrder(descending=True) + pod = to_pod(orderer) + self.assertEqual(pod["type"], "sorted") + result = from_pod(pod) + self.assertEqual(orderer, result) + + +class TestPackageOrderListCoverage(_BaseTestPackagesOrder): + """Coverage for PackageOrderList methods.""" + + def test_dirty_flag_on_append(self) -> None: + """append sets dirty flag.""" + ol = PackageOrderList() + ol.append(self.SortedOrder(descending=True)) + self.assertTrue(ol.dirty) + + def test_dirty_flag_on_extend(self) -> None: + """extend sets dirty flag.""" + ol = PackageOrderList() + ol.extend([self.SortedOrder(descending=True)]) + self.assertTrue(ol.dirty) + + def test_dirty_flag_on_pop(self) -> None: + """pop sets dirty flag.""" + ol = PackageOrderList([self.SortedOrder(descending=True)]) + ol.pop() + self.assertTrue(ol.dirty) + + def test_dirty_flag_on_remove(self) -> None: + """remove sets dirty flag.""" + orderer = self.SortedOrder(descending=True) + ol = PackageOrderList([orderer]) + ol.remove(orderer) + self.assertTrue(ol.dirty) + + def test_dirty_flag_on_clear(self) -> None: + """clear sets dirty flag.""" + ol = PackageOrderList([self.SortedOrder(descending=True)]) + ol.clear() + self.assertTrue(ol.dirty) + + def test_dirty_flag_on_insert(self) -> None: + """insert sets dirty flag.""" + ol = PackageOrderList() + ol.insert(0, self.SortedOrder(descending=True)) + self.assertTrue(ol.dirty) + + def test_to_orderer_with_dict(self) -> None: + """_to_orderer converts a dict to an orderer via from_pod.""" + ol = PackageOrderList() + orderer = ol._to_orderer({"type": "sorted", "descending": True}) + self.assertTrue(orderer.descending) + + def test_pod_round_trip(self) -> None: + """PackageOrderList to_pod/from_pod round-trip.""" + ol = PackageOrderList([self.SortedOrder(descending=True)]) + pod = ol.to_pod() + result = PackageOrderList.from_pod(pod) + self.assertEqual(ol, result) + + def test_get_with_dirty_refresh(self) -> None: + """get() triggers refresh when dirty.""" + orderer = self.SortedOrder(descending=True, packages=["foo"]) + ol = PackageOrderList([orderer]) + self.assertTrue(ol.dirty) + result = ol.get("foo") + self.assertEqual(result, orderer) + self.assertFalse(ol.dirty) + + def test_get_missing_package_returns_default(self) -> None: + """get() returns default for unknown package.""" + ol = PackageOrderList() + result = ol.get("nonexistent", default=self.NullPackageOrder()) + self.assertIsInstance(result, self.NullPackageOrder) diff --git a/src/rezplugins/package_order/__init__.py b/src/rezplugins/package_order/__init__.py new file mode 100644 index 0000000000..81d155de26 --- /dev/null +++ b/src/rezplugins/package_order/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from rez.plugin_managers import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/src/rezplugins/package_order/no_order.py b/src/rezplugins/package_order/no_order.py new file mode 100644 index 0000000000..d3263be454 --- /dev/null +++ b/src/rezplugins/package_order/no_order.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from rez.package_order import PackageOrder +from rez.utils.typing import SupportsLessThan +from rez.version import Version + +if TYPE_CHECKING: + from typing import Self + + +class NullPackageOrder(PackageOrder): + """An orderer that does not change the order - a no op. + + This orderer is useful in cases where you want to apply some default orderer + to a set of packages, but may want to explicitly NOT reorder a particular + package. You would use a :class:`NullPackageOrder` in a :class:`PerFamilyOrder` to do this. + """ + + name = "no_order" + + def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: + # python's sort will preserve the order of items that compare equal, so + # to not change anything, we just return the same object for all... + return 0 + + def __str__(self) -> str: + return "{}" + + def __eq__(self, other): + return type(self) is type(other) + + def to_pod(self) -> dict[str, object]: + """ + Example (in yaml): + + .. code-block:: yaml + + type: no_order + packages: ["foo"] + """ + return { + "packages": self.packages, + } + + @classmethod + def from_pod(cls, data: dict[str, object]) -> Self: + return cls(packages=data.get("packages")) + + +def register_plugin(): + return NullPackageOrder diff --git a/src/rezplugins/package_order/per_family.py b/src/rezplugins/package_order/per_family.py new file mode 100644 index 0000000000..86627067cf --- /dev/null +++ b/src/rezplugins/package_order/per_family.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import annotations + +from typing import Any, Callable, Iterable, TYPE_CHECKING + +from rez.package_order import PackageOrder, to_pod, from_pod +from rez.packages import Package +from rez.utils.typing import SupportsLessThan +from rez.version import Version + +if TYPE_CHECKING: + from typing import Self + + +class PerFamilyOrder(PackageOrder): + """An orderer that applies different orderers to different package families.""" + + name = "per_family" + + def __init__(self, order_dict: dict[str, PackageOrder], default_order: PackageOrder | None = None) -> None: + """Create a reorderer. + + Args: + order_dict (dict[str, PackageOrder]): Orderers to apply to + each package family. + default_order (PackageOrder): Orderer to apply to any packages + not specified in ``order_dict``. + """ + super().__init__(list(order_dict)) + self.order_dict = order_dict.copy() + self.default_order = default_order + + def reorder(self, iterable: Iterable[Package], key: Callable[[Any], Package] | None = None) -> list[Package] | None: + package_name = self._get_package_name_from_iterable(iterable, key) + if package_name is None: + return None + + orderer = self.order_dict.get(package_name) + if orderer is None: + orderer = self.default_order + if orderer is None: + return None + + return orderer.reorder(iterable, key) + + def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: + orderer = self.order_dict.get(package_name) + if orderer is None: + if self.default_order is None: + # shouldn't get here, because applies_to should protect us... + raise RuntimeError( + "package family orderer %r does not apply to package family %r", (self, package_name) + ) + + orderer = self.default_order + + return orderer.sort_key_implementation(package_name, version) + + def __str__(self) -> str: + items = sorted((x[0], str(x[1])) for x in self.order_dict.items()) + return str((items, str(self.default_order))) + + def __eq__(self, other): + return ( + type(other) is type(self) + and self.order_dict == other.order_dict + and self.default_order == other.default_order + ) + + def to_pod(self) -> dict[str, Any]: + """ + Example (in yaml): + + .. code-block:: yaml + + type: per_family + orderers: + - packages: ['foo', 'bah'] + type: version_split + first_version: '4.0.5' + - packages: ['python'] + type: sorted + descending: false + default_order: + type: sorted + descending: true + """ + orderers = {} + packages = {} + + # group package fams by orderer they use + for fam, orderer in self.order_dict.items(): + k = id(orderer) + orderers[k] = orderer + packages.setdefault(k, set()).add(fam) + + orderlist = [] + for k, fams in packages.items(): + orderer = orderers[k] + data = to_pod(orderer) + data["packages"] = sorted(fams) + orderlist.append(data) + + result: dict[str, Any] = {"orderers": orderlist} + + if self.default_order is not None: + result["default_order"] = to_pod(self.default_order) + + return result + + @classmethod + def from_pod(cls, data: dict[str, Any]) -> Self: + order_dict = {} + default_order = None + + for d in data["orderers"]: + d = d.copy() + fams = d.pop("packages") + orderer = from_pod(d) + + for fam in fams: + order_dict[fam] = orderer + + d = data.get("default_order") + if d: + default_order = from_pod(d) + + return cls(order_dict, default_order) + + +def register_plugin(): + return PerFamilyOrder diff --git a/src/rezplugins/package_order/soft_timestamp.py b/src/rezplugins/package_order/soft_timestamp.py new file mode 100644 index 0000000000..aab5f3b0b6 --- /dev/null +++ b/src/rezplugins/package_order/soft_timestamp.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from rez.package_order import PackageOrder +from rez.packages import iter_packages +from rez.utils.typing import SupportsLessThan +from rez.version import Version +from rez.version._version import _ReversedComparable + +if TYPE_CHECKING: + from typing import Self + + +class TimestampPackageOrder(PackageOrder): + """A timestamp order function. + + Given a time ``T``, this orderer returns packages released before ``T``, in descending + order, followed by those released after. If ``rank`` is non-zero, version + changes at that rank and above are allowed over the timestamp. + + For example, consider the common case where we want to prioritize packages + released before ``T``, except for newer patches. Consider the following package + versions, and time ``T``: + + .. code-block:: text + + 2.2.1 + 2.2.0 + 2.1.1 + 2.1.0 + 2.0.6 + 2.0.5 + <-- T + 2.0.0 + 1.9.0 + + A timestamp orderer set to ``rank=3`` (patch versions) will attempt to consume + the packages in the following order: + + .. code-block:: text + + 2.0.6 + 2.0.5 + 2.0.0 + 1.9.0 + 2.1.1 + 2.1.0 + 2.2.1 + 2.2.0 + + Notice that packages before ``T`` are preferred, followed by newer versions. + Newer versions are consumed in ascending order, except within rank (this is + why 2.1.1 is consumed before 2.1.0). + """ + + name = "soft_timestamp" + + def __init__(self, timestamp: int, rank: int = 0, packages: list[str] | None = None) -> None: + """Create a reorderer. + + Args: + timestamp (int): Epoch time of timestamp. Packages before this time + are preferred. + rank (int): If non-zero, allow version changes at this rank or above + past the timestamp. + """ + super().__init__(packages) + self.timestamp = timestamp + self.rank = rank + + # dictionary mapping from package family to the first-version-after + # the given timestamp + self._cached_first_after = {} + self._cached_sort_key = {} + + def _get_first_after(self, package_family: str) -> Version | None: + """Get the first package version that is after the timestamp""" + try: + first_after = self._cached_first_after[package_family] + except KeyError: + first_after = self._calc_first_after(package_family) + self._cached_first_after[package_family] = first_after + return first_after + + def _calc_first_after(self, package_family: str) -> Version | None: + descending = sorted(iter_packages(package_family), key=lambda p: p.version, reverse=True) + first_after = None + for i, package in enumerate(descending): + if not package.timestamp: + continue + if package.timestamp > self.timestamp: + first_after = package.version + else: + break + + if not self.rank: + return first_after + + # if we have rank, then we need to then go back UP the + # versions, until we find one whose trimmed version doesn't + # match. + # Note that we COULD do this by simply iterating through + # an ascending sequence, in which case we wouldn't have to + # "switch direction" after finding the first result after + # by timestamp... but we're making the assumption that the + # timestamp break will be closer to the higher end of the + # version, and that we'll therefore have to check fewer + # timestamps this way... + trimmed_version = package.version.trim(self.rank - 1) + first_after = None + for after_package in reversed(descending[:i]): + if after_package.version.trim(self.rank - 1) != trimmed_version: + return after_package.version + + return first_after + + def _calc_sort_key(self, package_name: str, version: Version) -> SupportsLessThan: + first_after = self._get_first_after(package_name) + if first_after is None: + # all packages are before T + is_before: bool | int = True + else: + is_before = int(version < first_after) + + if is_before: + return is_before, version + + if self.rank: + return (is_before, _ReversedComparable(version.trim(self.rank - 1)), version.tokens[self.rank - 1:]) + + return is_before, _ReversedComparable(version) + + def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: + cache_key = (package_name, str(version)) + result = self._cached_sort_key.get(cache_key) + if result is None: + result = self._calc_sort_key(package_name, version) + self._cached_sort_key[cache_key] = result + + return result + + def __str__(self) -> str: + return str((self.timestamp, self.rank)) + + def __eq__(self, other): + return type(other) is type(self) and self.timestamp == other.timestamp and self.rank == other.rank + + def to_pod(self) -> dict[str, Any]: + """ + Example (in yaml): + + .. code-block:: yaml + + type: soft_timestamp + timestamp: 1234567 + rank: 3 + packages: ["foo"] + """ + return dict( + timestamp=self.timestamp, + rank=self.rank, + packages=self.packages, + ) + + @classmethod + def from_pod(cls, data: dict[str, Any]) -> Self: + return cls( + data["timestamp"], + rank=data.get("rank", 0), + packages=data.get("packages"), + ) + + +def register_plugin(): + return TimestampPackageOrder diff --git a/src/rezplugins/package_order/sorted.py b/src/rezplugins/package_order/sorted.py new file mode 100644 index 0000000000..e226727a41 --- /dev/null +++ b/src/rezplugins/package_order/sorted.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from rez.package_order import PackageOrder +from rez.utils.typing import SupportsLessThan +from rez.version import Version +from rez.version._version import _ReversedComparable + +if TYPE_CHECKING: + from typing import Self + + +class SortedOrder(PackageOrder): + """An orderer that sorts based on :attr:`Package.version `.""" + + name = "sorted" + + def __init__(self, descending: bool, packages: list[str] | None = None) -> None: + super().__init__(packages) + self.descending = descending + + def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: + # Note that the name "descending" can be slightly confusing - it + # indicates that the final ordering this Order gives should be + # version descending (ie, the default) - however, the sort_key itself + # returns its results in "normal" ascending order (because it needs to + # be used "alongside" normally-sorted objects like versions). + # when the key is passed to sort(), though, it is always invoked with + # reverse=True... + if self.descending: + return version + else: + return _ReversedComparable(version) + + def __str__(self) -> str: + return str(self.descending) + + def __eq__(self, other): + return type(self) is type(other) and self.descending == other.descending + + def to_pod(self) -> dict[str, object]: + """ + Example (in yaml): + + .. code-block:: yaml + + type: sorted + descending: true + packages: ["foo"] + """ + return { + "descending": self.descending, + "packages": self.packages, + } + + @classmethod + def from_pod(cls, data: dict[str, object]) -> Self: + return cls( + data["descending"], + packages=data.get("packages"), + ) + + +def register_plugin(): + return SortedOrder diff --git a/src/rezplugins/package_order/version_split.py b/src/rezplugins/package_order/version_split.py new file mode 100644 index 0000000000..7ba3b71194 --- /dev/null +++ b/src/rezplugins/package_order/version_split.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from rez.package_order import PackageOrder +from rez.utils.typing import SupportsLessThan +from rez.version import Version + +if TYPE_CHECKING: + from typing import Self + + +class VersionSplitPackageOrder(PackageOrder): + """Orders package versions <= a given version first. + + For example, given the versions [5, 4, 3, 2, 1], an orderer initialized + with ``version=3`` would give the order [3, 2, 1, 5, 4]. + """ + + name = "version_split" + + def __init__(self, first_version: Version, packages: list[str] | None = None) -> None: + """Create a reorderer. + + Args: + first_version (Version): Start with versions <= this value. + """ + super().__init__(packages) + self.first_version = first_version + + def sort_key_implementation(self, package_name: str, version: Version) -> SupportsLessThan: + priority_key = 1 if version <= self.first_version else 0 + return priority_key, version + + def __str__(self) -> str: + return str(self.first_version) + + def __eq__(self, other): + return type(other) is type(self) and self.first_version == other.first_version + + def to_pod(self) -> dict[str, object]: + """ + Example (in yaml): + + .. code-block:: yaml + + type: version_split + first_version: "3.0.0" + packages: ["foo"] + """ + return dict( + first_version=str(self.first_version), + packages=self.packages, + ) + + @classmethod + def from_pod(cls, data: dict[str, object]) -> Self: + return cls( + Version(data["first_version"]), + packages=data.get("packages"), + ) + + +def register_plugin(): + return VersionSplitPackageOrder