diff --git a/examples/template/build_template_example.py b/examples/template/build_template_example.py deleted file mode 100644 index 7572cb83..00000000 --- a/examples/template/build_template_example.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Load and print templated devices; run from this example's directory.""" - -from pyaml.accelerator import Accelerator - -sr = Accelerator.load("templated_config.yaml") -for dev in sr._devices: - print(dev) diff --git a/examples/template/templated_config.yaml b/examples/template/templated_config.yaml deleted file mode 100644 index 71be2cc3..00000000 --- a/examples/template/templated_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -class: pyaml.accelerator.Accelerator -machine: sr -facility: PETRAIII -energy: 6e9 -controls: - - class: tango.pyaml.controlsystem.TangoControlSystem - tango_host: ebs-simu-3:10000 - name: live - catalog: - class: tango.pyaml.tango_catalog.TangoCatalog - disconnected: True -devices: -- ${template:SHI,SHI,01} -- ${template:SHI,SHI,02} -- ${template:SHI,SHI,03} -templates: - - name: SHI - parameters: - - sh_name - - ii - config: - class: pyaml.magnet.hcorrector.HCorrector - name: "{sh_name}-{ii}" - model: - class: pyaml.magnet.linear_model.LinearMagnetModel - unit: rad - hardware_unit: str - calibration_factor: 1.0 - powerconverter: tango/MAGNET/{sh_name}-{ii}/B1L diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 10ace630..c8b09aa9 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,9 +1,8 @@ """ Load PyAML configuration files and expand nested references. -The loader supports YAML and JSON files, parameterized templates, environment -and path resolvers, recursive file includes, and optional source-location -metadata for diagnostics. +The loader supports YAML and JSON files, environment and path resolvers, +recursive file includes, and optional source-location metadata for diagnostics. """ import io @@ -24,7 +23,6 @@ from yaml.loader import SafeLoader from .. import PyAMLException -from .template import TemplateManager logger = logging.getLogger(__name__) @@ -139,11 +137,6 @@ class LoadContext: Preserve source locations in loaded mappings. include_stack : list[pathlib.Path], optional Active include chain. Usually left empty for a new session. - expand : callable or None, optional - Callback used to expand generated configurations. Installed temporarily - by the active file loader and restored when that loader finishes. - templates : TemplateManager, optional - Template registry shared by files in this session. Defaults to a fresh registry. Methods ------- @@ -153,9 +146,6 @@ class LoadContext: include_locations: bool = False include_stack: list[Path] = field(default_factory=list) - expand: Callable[[Any], Any] | None = None - # Is populated with ConfigLoader's expand function for use in template resolver - templates: TemplateManager = field(default_factory=TemplateManager) @contextmanager def loading(self, path: Path): @@ -302,86 +292,16 @@ def resolve_include(value: str, context: LoadContext | None = None) -> Any: return _load(value, context) -@resolver("template") -def resolve_template(value: str, context: LoadContext | None = None) -> Any: +def load(filename: str, include_locations: bool = False) -> Union[dict, list]: """ - Instantiate a template and recursively expand its configuration. + Load a configuration file. - Parameters - ---------- - value : str - Name and comma-separated string arguments in the format - ``NAME,ARG1,ARG2,...``, or ``NAME`` for no arguments. A trailing comma - supplies an empty-string argument. Commas in arguments are not escaped. - context : LoadContext or None, optional - Active loading context providing the template registry and expansion - callback. Required for resolution despite the compatibility default. - - Returns - ------- - object - Parsed and expanded configuration data. - - Raises - ------ - PyAMLException - If the active context is missing, the template name is unknown, or - expansion exceeds the recursion limit. Expansion errors also propagate. - PyAMLConfigException - If the number of arguments does not match the template parameters. - """ - if context is None or context.expand is None: - raise PyAMLException("Template resolver requires an active loading context.") - - try: - name, separator, argument_text = value.partition(",") - arguments = argument_text.split(",") if separator else [] - try: - generated = context.templates.generate(name, *arguments) - return context.expand(generated) - except RecursionError as exc: - raise PyAMLException( - f"Recursion limit reached while expanding template {name!r}. " - "Check for circular template references or excessive nesting." - ) from exc - except KeyError as exc: - raise PyAMLException(f"Invalid template resolver call {value}.") from exc - - -def load(filename: str, include_locations: bool = False, *, templates: TemplateManager | None = None) -> Union[dict, list]: - """ - Load a configuration file, register templates, and expand references. - - Parameters - ---------- - filename : str - YAML or JSON filename. Relative paths are resolved against ``ROOT``. - include_locations : bool, optional - Preserve source-location metadata in YAML mappings. Defaults to False, - which uses the faster C-based YAML loader without location metadata. - templates : TemplateManager or None, optional - Registry to share across configuration fragments. If omitted, a fresh - registry is created. Included files share the same registry. - - Returns - ------- - dict or list - Expanded configuration with root-level template definitions removed. - - Raises - ------ - PyAMLException - If the file format is unsupported, parsing fails, or reference - expansion fails. - PyAMLConfigException - If template registration or argument-count validation fails. + When include_locations is False, uses the faster C-based YAML loader + and skips including source location metadata. """ # Create a new context - context = LoadContext( - include_locations=include_locations, - templates=templates if templates is not None else TemplateManager(), - ) + context = LoadContext(include_locations=include_locations) return _load(filename, context) @@ -399,12 +319,7 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]: else: raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") - previous_expand = context.expand - context.expand = loader.expand - try: - return loader.load() - finally: - context.expand = previous_expand + return loader.load() class ConfigLoader(ABC): @@ -422,8 +337,6 @@ class ConfigLoader(ABC): ------- expand(obj) Recursively expand configuration values. - register_templates(config) - Register and remove root-level template definitions before expansion. load() Load and parse the current configuration file. """ @@ -586,34 +499,6 @@ def _expand_list(self, items: list) -> list: return expanded - def register_templates(self, config): - """ - Register root-level templates in the current loading context. - - Parameters - ---------- - config : object - Parsed configuration. For dictionaries, the ``templates`` section - is removed in place and its definitions are registered. Other - values are left unchanged. - - Raises - ------ - PyAMLConfigException - If a template name is already registered in this context's registry. - KeyError - If a definition lacks ``name``, ``parameters``, or ``config``. - """ - if not isinstance(config, dict): - return - - for template in config.pop("templates", []): - self.context.templates.add( - name=template["name"], - parameters=template["parameters"], - config=template["config"], - ) - @abstractmethod def load(self) -> Union[dict, list]: """Load and parse the current configuration file.""" @@ -634,7 +519,7 @@ class YAMLLoader(ConfigLoader): Methods ------- load() - Parse YAML, register root-level templates, and expand references. + Parse the YAML file and expand nested configuration references. """ def __init__(self, path: Path, context: LoadContext): @@ -644,15 +529,12 @@ def __init__(self, path: Path, context: LoadContext): self._loader = SafeLineLoader if context.include_locations else CLoader def load(self) -> Union[dict, list]: - """Parse YAML, register root-level templates, and expand configuration references.""" + """Parse the YAML file and expand nested configuration references.""" logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") with open(self.path) as file: try: - parsed_config = yaml.load(file, Loader=self._loader) - # "templates" is popped out here if it exists - self.register_templates(parsed_config) - return self.expand(parsed_config) + return self.expand(yaml.load(file, Loader=self._loader)) except yaml.YAMLError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc @@ -671,7 +553,7 @@ class JSONLoader(ConfigLoader): Methods ------- load() - Parse JSON, register root-level templates, and expand references. + Parse the JSON file and expand nested configuration references. """ def __init__(self, path: Path, context: LoadContext): @@ -680,15 +562,12 @@ def __init__(self, path: Path, context: LoadContext): super().__init__(path, context) def load(self) -> Union[dict, list]: - """Parse JSON, register root-level templates, and expand configuration references.""" + """Parse the JSON file and expand nested configuration references.""" logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") with open(self.path) as file: try: - parsed_config = json.load(file) - # "templates" is popped out here if it exists - self.register_templates(parsed_config) - return self.expand(parsed_config) + return self.expand(json.load(file)) except json.JSONDecodeError as exc: raise PyAMLException(f"{self.path}: {exc}") from exc diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 94998b94..68baf41c 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -24,7 +24,6 @@ from ..common.exception import PyAMLConfigException from .fileloader import ROOT, load from .restfetcher import REMOTE_BASE_URL_KEY, SourceRoot, fetch_remote_config, is_remote_url, resolve_reference -from .template import TemplateManager _INTERNAL_METADATA_KEYS = {"__location__", "__fieldlocations__", REMOTE_BASE_URL_KEY} _VALID_QUERY_KEY_RE = re.compile(r"^[A-Z0-9_]+$") @@ -60,7 +59,7 @@ class ConfigurationManager: replace(category, element) Replace an existing named entry in an aggregated category. clear(category=None) - Clear the aggregated state and templates, or a single root field/category. + Clear the aggregated state, or a single root field/category. categories() Return categories that currently contain entries. keys(category=None) @@ -157,8 +156,7 @@ def __init__(self): Initialize an empty configuration manager. The manager starts with the default accelerator type and empty named - categories. Source tracking is enabled as fragments are added. Local - file fragments share this manager's private template registry. + categories. Source tracking is enabled as fragments are added. """ self._state: dict[str, Any] = {"class_path": self.DEFAULT_CLASS_PATH} self._items_by_category: dict[str, dict[str, dict[str, Any]]] = {category: {} for category in self.NAMED_CATEGORIES} @@ -166,7 +164,6 @@ def __init__(self): self._field_sources: dict[str, str] = {} self._build_root: SourceRoot = ROOT.get() self._build_root_locked = False - self._templates = TemplateManager() def add(self, payload, **kwargs) -> None: r""" @@ -290,14 +287,12 @@ def replace(self, category: str, element: dict) -> None: def clear(self, category: str | None = None) -> None: r""" - Clear all configuration state and templates, or one root field/category. + Clear the aggregated state, or a single root field/category. Parameters ---------- category : str, optional - If provided, only that category or root field is cleared and - template definitions are retained. If omitted, all aggregated - state and this manager's template registry are cleared. + If provided, only that category or root field is cleared. Examples -------- @@ -311,7 +306,6 @@ def clear(self, category: str | None = None) -> None: >>> manager.clear() """ if category is None: - self._templates.clear() self._state = {"class_path": self.DEFAULT_CLASS_PATH} self._field_sources.clear() for name in self.NAMED_CATEGORIES: @@ -704,7 +698,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, include_locations, templates=self._templates) + fragment = load(resolved_path.name, include_locations) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/template.py b/pyaml/configuration/template.py deleted file mode 100644 index 7cf5d2a9..00000000 --- a/pyaml/configuration/template.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Store and instantiate parameterized configuration templates. - -Each registry owns its definitions. Generation substitutes positional arguments -into a copy of the template; the configuration loader expands any remaining -file, environment, or template references. -""" - -import copy -import logging -import re -from collections.abc import Mapping -from typing import Any - -from ..common.exception import PyAMLConfigException - -logger = logging.getLogger(__name__) - - -def substitute(obj: Any, arguments: Mapping[str, object]) -> Any: - """ - Recursively substitute named placeholders in configuration values. - - Parameters - ---------- - obj : object - Configuration value to process. Dictionaries and lists are traversed; - dictionary keys and non-string scalar values are left unchanged. - arguments : Mapping[str, object] - Parameter names mapped to replacement values, converted to strings. - - Returns - ------- - object - Configuration with substitutions applied and new dictionaries and lists. - - Notes - ----- - Replacements use ``{name}`` placeholders and follow argument insertion order. - Text inserted by one replacement can be modified by a later replacement. - """ - if isinstance(obj, dict): - return {key: substitute(value, arguments) for key, value in obj.items()} - if isinstance(obj, list): - return [substitute(value, arguments) for value in obj] - if isinstance(obj, str): - for name, value in arguments.items(): - obj = obj.replace("{" + name + "}", str(value)) - return obj - return obj - - -class TemplateManager: - """ - Store template definitions for one configuration manager or loading session. - - Definitions are isolated between instances and copied on registration and - generation so callers can modify their configurations independently. - - Methods - ------- - add(name, parameters, config) - Register a named template with ordered parameters. - generate(name, *args) - Substitute positional arguments into a copy of a template. - clear() - Remove all definitions from this registry. - """ - - def __init__(self) -> None: - """Initialize an empty template registry.""" - self.template_parameters: dict[str, list[str]] = {} - self.template_configs: dict[str, dict[str, Any]] = {} - - def add(self, name: str, parameters: list[str], config: dict[str, Any]) -> None: - """ - Register a template, copying its parameters and configuration. - - Parameters - ---------- - name : str - Template name, unique within this registry. - parameters : list[str] - Parameter names in the order expected by :meth:`generate`. - config : dict[str, Any] - Configuration body containing ``{parameter}`` placeholders. - - Raises - ------ - PyAMLConfigException - If the name is already registered. - """ - if name in self.template_configs: - raise PyAMLConfigException(f"Template '{name}' has already been registered.") - - self.template_parameters[name] = list(parameters) - self.template_configs[name] = copy.deepcopy(config) - - def generate(self, name: str, *args: object) -> dict[str, Any]: - """ - Generate a configuration by substituting positional arguments. - - Parameters - ---------- - name : str - Name of a registered template. - *args : object - Values corresponding to the template's ordered parameters. - Values are converted to strings during substitution. - - Returns - ------- - dict[str, Any] - Independent configuration with placeholders replaced. Resolver - expressions and file references are left for the loader to expand. - - Raises - ------ - KeyError - If the template name is not registered. - PyAMLConfigException - If the number of arguments does not match the parameters. - - Notes - ----- - Arguments containing brace-delimited placeholders produce a logging - warning because sequential substitution may modify their contents. - """ - number_of_parameters = len(self.template_parameters[name]) - number_of_arguments = len(args) - - if number_of_parameters != number_of_arguments: - raise PyAMLConfigException( - f"Invalid number of arguments ({args}: {number_of_arguments}) passed to template {name}." - f" Expected {number_of_parameters}." - ) - - # name the arguments by position - arguments_dict: dict[str, object] = {} - for arg_name, arg_value in zip(self.template_parameters[name], args, strict=True): - # check if {...} is included in any of the arguments, and a raise a warning if so. - if re.search(r"\{[^{}]+\}", str(arg_value)): - logger.warning( - f"Argument {arg_name!r} for template {name!r} contains a placeholder: {arg_value!r}. " - "Sequential replacement may substitute placeholders inside this argument.", - ) - arguments_dict[arg_name] = arg_value - - config = substitute(copy.deepcopy(self.template_configs[name]), arguments_dict) - - return config - - def clear(self) -> None: - """Remove all definitions from this registry without affecting other instances.""" - self.template_parameters.clear() - self.template_configs.clear() diff --git a/tests/configuration/test_templates.py b/tests/configuration/test_templates.py deleted file mode 100644 index 712040b7..00000000 --- a/tests/configuration/test_templates.py +++ /dev/null @@ -1,240 +0,0 @@ -import json - -import pytest -import yaml - -from pyaml import PyAMLException -from pyaml.accelerator import Accelerator -from pyaml.common.exception import PyAMLConfigException -from pyaml.configuration import ConfigurationManager -from pyaml.configuration.fileloader import FIELD_LOCATIONS_KEY, ROOT, load -from pyaml.configuration.template import TemplateManager - - -def _template_definition(prefix=""): - return { - "name": "device", - "parameters": ["name"], - "config": {"class": "pyaml.magnet.hcorrector.HCorrector", "name": prefix + "{name}"}, - } - - -def _write_config(path, config): - path.write_text(json.dumps(config) if path.suffix == ".json" else yaml.safe_dump(config, sort_keys=False)) - return path - - -@pytest.fixture -def template_config_root(tmp_path): - previous_root = ROOT.get() - ROOT.set(tmp_path) - try: - yield tmp_path - finally: - ROOT.set(previous_root) - - -def test_template_definitions_are_independent_of_inputs_and_results(): - registry = TemplateManager() - parameters = ["name"] - config = {"name": "{name}", "nested": [1]} - registry.add("T", parameters, config) - - parameters.clear() - config["nested"].append(2) - - generated = registry.generate("T", "Q1") - assert generated == {"name": "Q1", "nested": [1]} - - generated["nested"].append(3) - - assert registry.generate("T", "Q2") == {"name": "Q2", "nested": [1]} - - -def test_template_registry_rejects_duplicate_names(): - registry = TemplateManager() - registry.add("T", ["name"], {"name": "{name}"}) - - with pytest.raises(PyAMLConfigException, match="Template 'T'.*registered"): - registry.add("T", [], {}) - - -@pytest.mark.parametrize("suffix", [".yaml", ".json"]) -def test_standalone_loads_have_fresh_template_registries(tmp_path, suffix): - path = _write_config( - tmp_path / ("config" + suffix), - {"templates": [_template_definition()], "devices": ["${template:device,Q1}"]}, - ) - - first = load(str(path)) - second = load(str(path)) - - assert first["devices"][0]["name"] == "Q1" - assert second["devices"][0]["name"] == "Q1" - - use_only = _write_config(tmp_path / "use.yaml", {"devices": ["${template:device,Q1}"]}) - with pytest.raises(PyAMLException, match="Invalid template resolver call"): - load(str(use_only)) - - -def test_configuration_managers_have_independent_templates(tmp_path): - first_definitions = _write_config(tmp_path / "first.yaml", {"templates": [_template_definition("first-")]}) - second_definitions = _write_config(tmp_path / "second.json", {"templates": [_template_definition("second-")]}) - usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) - first = ConfigurationManager() - second = ConfigurationManager() - - first.add(first_definitions) - second.add(second_definitions) - first.add(usage) - second.add(usage) - - assert first.keys("devices") == ["first-Q1"] - assert second.keys("devices") == ["second-Q1"] - - first.clear() - second.clear("devices") - second.add(usage) - - assert second.keys("devices") == ["second-Q1"] - - -def test_manager_category_clear_preserves_templates(tmp_path): - definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) - usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) - manager = ConfigurationManager() - manager.add(definitions) - manager.add(usage) - - manager.clear("devices") - manager.add(usage) - - assert manager.keys("devices") == ["Q1"] - - -def test_manager_clear_resets_template_registration(tmp_path): - definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) - usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) - manager = ConfigurationManager() - manager.add(definitions) - - manager.clear() - - with pytest.raises(PyAMLException, match="Invalid template resolver call"): - manager.add(usage) - - manager.add(definitions) - manager.add(usage) - - assert manager.keys("devices") == ["Q1"] - - -def test_included_files_and_nested_templates_share_registry(template_config_root, monkeypatch): - tmp_path = template_config_root - monkeypatch.setenv("PYAML_TEST_HOST", "localhost") - _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) - _write_config(tmp_path / "Q1.json", {"factor": 1.5}) - path = _write_config( - tmp_path / "parent.yaml", - { - "templates": [ - { - "name": "wrapper", - "parameters": ["name"], - "config": { - "device": "${template:device,{name}}", - "model": "{name}.json", - "host": "${env:PYAML_TEST_HOST}", - }, - } - ], - "definitions": "definitions.yaml", - "result": "${template:wrapper,Q1}", - }, - ) - - loaded = load(str(path), include_locations=True) - result = loaded["result"] - - assert result["device"]["name"] == "Q1" - assert result["model"] == {"factor": 1.5} - assert result["host"] == "localhost" - assert "templates" in loaded[FIELD_LOCATIONS_KEY] - assert "name" in result["device"][FIELD_LOCATIONS_KEY] - - -def test_explicit_registry_can_be_shared_between_standalone_loads(tmp_path): - registry = TemplateManager() - definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) - usage = _write_config(tmp_path / "usage.json", {"devices": ["${template:device,Q1}"]}) - - load(str(definitions), templates=registry) - result = load(str(usage), templates=registry) - - assert result["devices"][0]["name"] == "Q1" - - -def test_template_calls_distinguish_no_arguments_from_an_empty_argument(tmp_path): - path = _write_config( - tmp_path / "arguments.yaml", - { - "templates": [ - {"name": "constant", "parameters": [], "config": {"name": "Q1"}}, - _template_definition(), - ], - "constant": "${template:constant}", - "empty": "${template:device,}", - }, - ) - - result = load(str(path)) - - assert result["constant"]["name"] == "Q1" - assert result["empty"]["name"] == "" - - -def test_template_call_without_required_arguments_reports_count_error(tmp_path): - path = _write_config( - tmp_path / "missing_argument.yaml", - {"templates": [_template_definition()], "result": "${template:device}"}, - ) - - with pytest.raises(PyAMLConfigException, match=r"Invalid number of arguments.*Expected 1"): - load(str(path)) - - -def test_recursive_template_still_reports_configuration_error(tmp_path): - path = _write_config( - tmp_path / "recursive.yaml", - { - "templates": [{"name": "T", "parameters": ["name"], "config": {"child": "${template:T,{name}}"}}], - "result": "${template:T,Q1}", - }, - ) - - with pytest.raises(PyAMLException, match="Recursion limit reached while expanding template"): - load(str(path)) - - -def test_accelerator_load_does_not_clear_an_existing_managers_templates(template_config_root): - tmp_path = template_config_root - definitions = _write_config(tmp_path / "definitions.yaml", {"templates": [_template_definition()]}) - manager = ConfigurationManager() - manager.add(definitions) - accelerator = _write_config( - tmp_path / "accelerator.yaml", - { - "class": "pyaml.accelerator.Accelerator", - "facility": "test", - "machine": "sr", - "energy": 3e9, - "templates": [_template_definition("other-")], - }, - ) - usage = _write_config(tmp_path / "usage.yaml", {"devices": ["${template:device,Q1}"]}) - - for _ in range(2): - Accelerator.load(str(accelerator)) - manager.add(usage) - - assert manager.keys("devices") == ["Q1"]